diff options
Diffstat (limited to 'lib/Target/AArch64')
120 files changed, 8585 insertions, 2494 deletions
diff --git a/lib/Target/AArch64/AArch64.h b/lib/Target/AArch64/AArch64.h index c36d9354f3ba..6965403a25ab 100644 --- a/lib/Target/AArch64/AArch64.h +++ b/lib/Target/AArch64/AArch64.h @@ -1,9 +1,8 @@ //==-- AArch64.h - Top-level interface for AArch64 --------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -57,6 +56,7 @@ InstructionSelector * createAArch64InstructionSelector(const AArch64TargetMachine &, AArch64Subtarget &, AArch64RegisterBankInfo &); FunctionPass *createAArch64PreLegalizeCombiner(); +FunctionPass *createAArch64StackTaggingPass(); void initializeAArch64A53Fix835769Pass(PassRegistry&); void initializeAArch64A57FPLoadBalancingPass(PassRegistry&); @@ -79,6 +79,7 @@ void initializeAArch64StorePairSuppressPass(PassRegistry&); void initializeFalkorHWPFFixPass(PassRegistry&); void initializeFalkorMarkStridedAccessesLegacyPass(PassRegistry&); void initializeLDTLSCleanupPass(PassRegistry&); +void initializeAArch64StackTaggingPass(PassRegistry&); } // end namespace llvm #endif diff --git a/lib/Target/AArch64/AArch64.td b/lib/Target/AArch64/AArch64.td index 8f79140cba64..e39c6995e367 100644 --- a/lib/Target/AArch64/AArch64.td +++ b/lib/Target/AArch64/AArch64.td @@ -1,9 +1,8 @@ //=- AArch64.td - Describe the AArch64 Target Machine --------*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -104,6 +103,21 @@ def FeatureCCPP : SubtargetFeature<"ccpp", "HasCCPP", def FeatureSVE : SubtargetFeature<"sve", "HasSVE", "true", "Enable Scalable Vector Extension (SVE) instructions">; +def FeatureSVE2 : SubtargetFeature<"sve2", "HasSVE2", "true", + "Enable Scalable Vector Extension 2 (SVE2) instructions", [FeatureSVE]>; + +def FeatureSVE2AES : SubtargetFeature<"sve2-aes", "HasSVE2AES", "true", + "Enable AES SVE2 instructions", [FeatureSVE2, FeatureAES]>; + +def FeatureSVE2SM4 : SubtargetFeature<"sve2-sm4", "HasSVE2SM4", "true", + "Enable SM4 SVE2 instructions", [FeatureSVE2, FeatureSM4]>; + +def FeatureSVE2SHA3 : SubtargetFeature<"sve2-sha3", "HasSVE2SHA3", "true", + "Enable SHA3 SVE2 instructions", [FeatureSVE2, FeatureSHA3]>; + +def FeatureSVE2BitPerm : SubtargetFeature<"bitperm", "HasSVE2BitPerm", "true", + "Enable bit permutation SVE2 instructions", [FeatureSVE2]>; + def FeatureZCRegMove : SubtargetFeature<"zcm", "HasZeroCycleRegMove", "true", "Has zero-cycle register moves">; def FeatureZCZeroingGP : SubtargetFeature<"zcz-gp", "HasZeroCycleZeroingGP", "true", @@ -127,7 +141,7 @@ def FeatureStrictAlign : SubtargetFeature<"strict-align", "Disallow all unaligned memory " "access">; -foreach i = {1-7,18,20} in +foreach i = {1-7,9-15,18,20-28} in def FeatureReserveX#i : SubtargetFeature<"reserve-x"#i, "ReserveXRegister["#i#"]", "true", "Reserve X"#i#", making it unavailable " "as a GPR">; @@ -386,8 +400,28 @@ def AArch64InstrInfo : InstrInfo; include "AArch64SystemOperands.td" //===----------------------------------------------------------------------===// +// Access to privileged registers +//===----------------------------------------------------------------------===// + +foreach i = 1-3 in +def FeatureUseEL#i#ForTP : SubtargetFeature<"tpidr-el"#i, "UseEL"#i#"ForTP", + "true", "Permit use of TPIDR_EL"#i#" for the TLS base">; + +//===----------------------------------------------------------------------===// // AArch64 Processors supported. // + +//===----------------------------------------------------------------------===// +// Unsupported features to disable for scheduling models +//===----------------------------------------------------------------------===// + +class AArch64Unsupported { list<Predicate> F; } + +def SVEUnsupported : AArch64Unsupported { + let F = [HasSVE, HasSVE2, HasSVE2AES, HasSVE2SM4, HasSVE2SHA3, + HasSVE2BitPerm]; +} + include "AArch64SchedA53.td" include "AArch64SchedA57.td" include "AArch64SchedCyclone.td" @@ -483,6 +517,18 @@ def ProcA75 : SubtargetFeature<"a75", "ARMProcFamily", "CortexA75", FeaturePerfMon ]>; +def ProcA76 : SubtargetFeature<"a76", "ARMProcFamily", "CortexA76", + "Cortex-A76 ARM processors", [ + HasV8_2aOps, + FeatureFPARMv8, + FeatureNEON, + FeatureRCPC, + FeatureCrypto, + FeatureFullFP16, + FeatureDotProd, + FeatureSSBS + ]>; + // Note that cyclone does not fuse AES instructions, but newer apple chips do // perform the fusion and cyclone is used by default when targetting apple OSes. def ProcCyclone : SubtargetFeature<"cyclone", "ARMProcFamily", "Cyclone", @@ -554,7 +600,7 @@ def ProcExynosM4 : SubtargetFeature<"exynosm4", "ARMProcFamily", "ExynosM3", FeatureDotProd, FeatureExynosCheapAsMoveHandling, FeatureForce32BitJumpTables, - FeatureFP16FML, + FeatureFullFP16, FeatureFuseAddress, FeatureFuseAES, FeatureFuseArithmeticLogic, @@ -694,15 +740,17 @@ def : ProcessorModel<"cortex-a35", CortexA53Model, [ProcA35]>; def : ProcessorModel<"cortex-a53", CortexA53Model, [ProcA53]>; def : ProcessorModel<"cortex-a55", CortexA53Model, [ProcA55]>; def : ProcessorModel<"cortex-a57", CortexA57Model, [ProcA57]>; -// FIXME: Cortex-A72, Cortex-A73 and Cortex-A75 are currently modeled as a Cortex-A57. def : ProcessorModel<"cortex-a72", CortexA57Model, [ProcA72]>; def : ProcessorModel<"cortex-a73", CortexA57Model, [ProcA73]>; def : ProcessorModel<"cortex-a75", CortexA57Model, [ProcA75]>; +def : ProcessorModel<"cortex-a76", CortexA57Model, [ProcA76]>; +def : ProcessorModel<"cortex-a76ae", CortexA57Model, [ProcA76]>; def : ProcessorModel<"cyclone", CycloneModel, [ProcCyclone]>; def : ProcessorModel<"exynos-m1", ExynosM1Model, [ProcExynosM1]>; def : ProcessorModel<"exynos-m2", ExynosM1Model, [ProcExynosM2]>; def : ProcessorModel<"exynos-m3", ExynosM3Model, [ProcExynosM3]>; def : ProcessorModel<"exynos-m4", ExynosM4Model, [ProcExynosM4]>; +def : ProcessorModel<"exynos-m5", ExynosM4Model, [ProcExynosM4]>; def : ProcessorModel<"falkor", FalkorModel, [ProcFalkor]>; def : ProcessorModel<"saphira", FalkorModel, [ProcSaphira]>; def : ProcessorModel<"kryo", KryoModel, [ProcKryo]>; @@ -716,6 +764,9 @@ def : ProcessorModel<"thunderx2t99", ThunderX2T99Model, [ProcThunderX2T99]>; // FIXME: HiSilicon TSV110 is currently modeled as a Cortex-A57. def : ProcessorModel<"tsv110", CortexA57Model, [ProcTSV110]>; +// Alias for the latest Apple processor model supported by LLVM. +def : ProcessorModel<"apple-latest", CycloneModel, [ProcCyclone]>; + //===----------------------------------------------------------------------===// // Assembly parser //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64A53Fix835769.cpp b/lib/Target/AArch64/AArch64A53Fix835769.cpp index 30232afaf024..e80fe2cada09 100644 --- a/lib/Target/AArch64/AArch64A53Fix835769.cpp +++ b/lib/Target/AArch64/AArch64A53Fix835769.cpp @@ -1,9 +1,8 @@ //===-- AArch64A53Fix835769.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This pass changes code to work around Cortex-A53 erratum 835769. diff --git a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp index 452fbd3488b0..92c8c4955d50 100644 --- a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp +++ b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp @@ -1,9 +1,8 @@ //===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // For best-case performance on Cortex-A57, we should try to use a balanced diff --git a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp index 22b0c1e3b471..89404463e1f0 100644 --- a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp +++ b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp @@ -1,9 +1,8 @@ //===-- AArch64AdvSIMDScalar.cpp - Replace dead defs w/ zero reg --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // When profitable, replace GPR targeting i64 instructions with their diff --git a/lib/Target/AArch64/AArch64AsmPrinter.cpp b/lib/Target/AArch64/AArch64AsmPrinter.cpp index 0442076992e2..094fbd999523 100644 --- a/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -1,9 +1,8 @@ //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,10 +17,12 @@ #include "AArch64RegisterInfo.h" #include "AArch64Subtarget.h" #include "AArch64TargetObjectFile.h" -#include "InstPrinter/AArch64InstPrinter.h" #include "MCTargetDesc/AArch64AddressingModes.h" +#include "MCTargetDesc/AArch64InstPrinter.h" +#include "MCTargetDesc/AArch64MCExpr.h" #include "MCTargetDesc/AArch64MCTargetDesc.h" #include "MCTargetDesc/AArch64TargetStreamer.h" +#include "TargetInfo/AArch64TargetInfo.h" #include "Utils/AArch64BaseInfo.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" @@ -29,6 +30,7 @@ #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/COFF.h" +#include "llvm/BinaryFormat/ELF.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" @@ -44,6 +46,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" +#include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/Casting.h" @@ -96,6 +99,10 @@ public: void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); + std::map<std::pair<unsigned, uint32_t>, MCSymbol *> HwasanMemaccessSymbols; + void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); + void EmitHwasanMemaccessSymbols(Module &M); + void EmitSled(const MachineInstr &MI, SledKind Kind); /// tblgen'erated driver function for lowering simple MI->MC @@ -147,11 +154,9 @@ private: raw_ostream &O); bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &O) override; + const char *ExtraCode, raw_ostream &O) override; bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &O) override; + const char *ExtraCode, raw_ostream &O) override; void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS); @@ -230,7 +235,204 @@ void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind) recordSled(CurSled, MI, Kind); } +void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { + unsigned Reg = MI.getOperand(0).getReg(); + uint32_t AccessInfo = MI.getOperand(1).getImm(); + MCSymbol *&Sym = HwasanMemaccessSymbols[{Reg, AccessInfo}]; + if (!Sym) { + // FIXME: Make this work on non-ELF. + if (!TM.getTargetTriple().isOSBinFormatELF()) + report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); + + std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" + + utostr(AccessInfo); + Sym = OutContext.getOrCreateSymbol(SymName); + } + + EmitToStreamer(*OutStreamer, + MCInstBuilder(AArch64::BL) + .addExpr(MCSymbolRefExpr::create(Sym, OutContext))); +} + +void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) { + if (HwasanMemaccessSymbols.empty()) + return; + + const Triple &TT = TM.getTargetTriple(); + assert(TT.isOSBinFormatELF()); + std::unique_ptr<MCSubtargetInfo> STI( + TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); + + MCSymbol *HwasanTagMismatchSym = + OutContext.getOrCreateSymbol("__hwasan_tag_mismatch"); + + const MCSymbolRefExpr *HwasanTagMismatchRef = + MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext); + + for (auto &P : HwasanMemaccessSymbols) { + unsigned Reg = P.first.first; + uint32_t AccessInfo = P.first.second; + MCSymbol *Sym = P.second; + + OutStreamer->SwitchSection(OutContext.getELFSection( + ".text.hot", ELF::SHT_PROGBITS, + ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + Sym->getName())); + + OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); + OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak); + OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden); + OutStreamer->EmitLabel(Sym); + + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri) + .addReg(AArch64::X16) + .addReg(Reg) + .addImm(4) + .addImm(55), + *STI); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX) + .addReg(AArch64::W16) + .addReg(AArch64::X9) + .addReg(AArch64::X16) + .addImm(0) + .addImm(0), + *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::SUBSXrs) + .addReg(AArch64::XZR) + .addReg(AArch64::X16) + .addReg(Reg) + .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), + *STI); + MCSymbol *HandlePartialSym = OutContext.createTempSymbol(); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::Bcc) + .addImm(AArch64CC::NE) + .addExpr(MCSymbolRefExpr::create(HandlePartialSym, OutContext)), + *STI); + MCSymbol *ReturnSym = OutContext.createTempSymbol(); + OutStreamer->EmitLabel(ReturnSym); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); + + OutStreamer->EmitLabel(HandlePartialSym); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri) + .addReg(AArch64::WZR) + .addReg(AArch64::W16) + .addImm(15) + .addImm(0), + *STI); + MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::Bcc) + .addImm(AArch64CC::HI) + .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), + *STI); + + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::ANDXri) + .addReg(AArch64::X17) + .addReg(Reg) + .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), + *STI); + unsigned Size = 1 << (AccessInfo & 0xf); + if (Size != 1) + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(AArch64::X17) + .addReg(AArch64::X17) + .addImm(Size - 1) + .addImm(0), + *STI); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs) + .addReg(AArch64::WZR) + .addReg(AArch64::W16) + .addReg(AArch64::W17) + .addImm(0), + *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::Bcc) + .addImm(AArch64CC::LS) + .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), + *STI); + + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::ORRXri) + .addReg(AArch64::X16) + .addReg(Reg) + .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), + *STI); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui) + .addReg(AArch64::W16) + .addReg(AArch64::X16) + .addImm(0), + *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::SUBSXrs) + .addReg(AArch64::XZR) + .addReg(AArch64::X16) + .addReg(Reg) + .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), + *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::Bcc) + .addImm(AArch64CC::EQ) + .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), + *STI); + + OutStreamer->EmitLabel(HandleMismatchSym); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) + .addReg(AArch64::X0) + .addReg(AArch64::X1) + .addReg(AArch64::SP) + .addImm(-32), + *STI); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXi) + .addReg(AArch64::FP) + .addReg(AArch64::LR) + .addReg(AArch64::SP) + .addImm(29), + *STI); + + if (Reg != AArch64::X0) + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs) + .addReg(AArch64::X0) + .addReg(AArch64::XZR) + .addReg(Reg) + .addImm(0), + *STI); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi) + .addReg(AArch64::X1) + .addImm(AccessInfo) + .addImm(0), + *STI); + + // Intentionally load the GOT entry and branch to it, rather than possibly + // late binding the function, which may clobber the registers before we have + // a chance to save them. + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::ADRP) + .addReg(AArch64::X16) + .addExpr(AArch64MCExpr::create( + HwasanTagMismatchRef, + AArch64MCExpr::VariantKind::VK_GOT_PAGE, OutContext)), + *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::LDRXui) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addExpr(AArch64MCExpr::create( + HwasanTagMismatchRef, + AArch64MCExpr::VariantKind::VK_GOT_LO12, OutContext)), + *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI); + } +} + void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) { + EmitHwasanMemaccessSymbols(M); + const Triple &TT = TM.getTargetTriple(); if (TT.isOSBinFormatMachO()) { // Funny Darwin hack: This flag tells the linker that no global symbols @@ -295,14 +497,7 @@ void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum, break; } case MachineOperand::MO_GlobalAddress: { - const GlobalValue *GV = MO.getGlobal(); - MCSymbol *Sym = getSymbol(GV); - - // FIXME: Can we get anything other than a plain symbol here? - assert(!MO.getTargetFlags() && "Unknown operand target flag!"); - - Sym->print(O, MAI); - printOffset(MO.getOffset(), O); + PrintSymbolOperand(MO, O); break; } case MachineOperand::MO_BlockAddress: { @@ -348,12 +543,11 @@ bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO, } bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, - unsigned AsmVariant, const char *ExtraCode, raw_ostream &O) { const MachineOperand &MO = MI->getOperand(OpNum); // First try the generic code, which knows about modifiers like 'c' and 'n'. - if (!AsmPrinter::PrintAsmOperand(MI, OpNum, AsmVariant, ExtraCode, O)) + if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O)) return false; // Does this asm operand have a single letter operand modifier? @@ -364,9 +558,6 @@ bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, switch (ExtraCode[0]) { default: return true; // Unknown modifier. - case 'a': // Print 'a' modifier - PrintAsmMemoryOperand(MI, OpNum, AsmVariant, ExtraCode, O); - return false; case 'w': // Print W register case 'x': // Print X register if (MO.isReg()) @@ -432,7 +623,6 @@ bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, - unsigned AsmVariant, const char *ExtraCode, raw_ostream &O) { if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a') @@ -471,9 +661,18 @@ void AArch64AsmPrinter::EmitJumpTableInfo() { const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); if (JT.empty()) return; + const Function &F = MF->getFunction(); const TargetLoweringObjectFile &TLOF = getObjFileLowering(); - MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM); - OutStreamer->SwitchSection(ReadOnlySec); + bool JTInDiffSection = + !STI->isTargetCOFF() || + !TLOF.shouldPutJumpTableInFunctionSection( + MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32, + F); + if (JTInDiffSection) { + // Drop it in the readonly section. + MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM); + OutStreamer->SwitchSection(ReadOnlySec); + } auto AFI = MF->getInfo<AArch64FunctionInfo>(); for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) { @@ -694,6 +893,34 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) { switch (MI->getOpcode()) { default: break; + case AArch64::MOVMCSym: { + unsigned DestReg = MI->getOperand(0).getReg(); + const MachineOperand &MO_Sym = MI->getOperand(1); + MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym); + MCOperand Hi_MCSym, Lo_MCSym; + + Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S); + Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC); + + MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym); + MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym); + + MCInst MovZ; + MovZ.setOpcode(AArch64::MOVZXi); + MovZ.addOperand(MCOperand::createReg(DestReg)); + MovZ.addOperand(Hi_MCSym); + MovZ.addOperand(MCOperand::createImm(16)); + EmitToStreamer(*OutStreamer, MovZ); + + MCInst MovK; + MovK.setOpcode(AArch64::MOVKXi); + MovK.addOperand(MCOperand::createReg(DestReg)); + MovK.addOperand(MCOperand::createReg(DestReg)); + MovK.addOperand(Lo_MCSym); + MovK.addOperand(MCOperand::createImm(0)); + EmitToStreamer(*OutStreamer, MovK); + return; + } case AArch64::MOVIv2d_ns: // If the target has <rdar://problem/16473581>, lower this // instruction to movi.16b instead. @@ -856,6 +1083,10 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) { LowerPATCHABLE_TAIL_CALL(*MI); return; + case AArch64::HWASAN_CHECK_MEMACCESS: + LowerHWASAN_CHECK_MEMACCESS(*MI); + return; + case AArch64::SEH_StackAlloc: TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm()); return; diff --git a/lib/Target/AArch64/AArch64BranchTargets.cpp b/lib/Target/AArch64/AArch64BranchTargets.cpp index da70a624c5be..6fa3a462bc71 100644 --- a/lib/Target/AArch64/AArch64BranchTargets.cpp +++ b/lib/Target/AArch64/AArch64BranchTargets.cpp @@ -1,9 +1,8 @@ //===-- AArch64BranchTargets.cpp -- Harden code using v8.5-A BTI extension -==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64CallLowering.cpp b/lib/Target/AArch64/AArch64CallLowering.cpp index 5980e5684e89..59757769c89a 100644 --- a/lib/Target/AArch64/AArch64CallLowering.cpp +++ b/lib/Target/AArch64/AArch64CallLowering.cpp @@ -1,9 +1,8 @@ //===--- AArch64CallLowering.cpp - Call lowering --------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -45,6 +44,8 @@ #include <cstdint> #include <iterator> +#define DEBUG_TYPE "aarch64-call-lowering" + using namespace llvm; AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI) @@ -56,18 +57,18 @@ struct IncomingArgHandler : public CallLowering::ValueHandler { CCAssignFn *AssignFn) : ValueHandler(MIRBuilder, MRI, AssignFn), StackUsed(0) {} - unsigned getStackAddress(uint64_t Size, int64_t Offset, + Register getStackAddress(uint64_t Size, int64_t Offset, MachinePointerInfo &MPO) override { auto &MFI = MIRBuilder.getMF().getFrameInfo(); int FI = MFI.CreateFixedObject(Size, Offset, true); MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); - unsigned AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64)); + Register AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64)); MIRBuilder.buildFrameIndex(AddrReg, FI); StackUsed = std::max(StackUsed, Size + Offset); return AddrReg; } - void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign &VA) override { markPhysRegUsed(PhysReg); switch (VA.getLocInfo()) { @@ -84,11 +85,12 @@ struct IncomingArgHandler : public CallLowering::ValueHandler { } } - void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size, + void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) override { + // FIXME: Get alignment auto MMO = MIRBuilder.getMF().getMachineMemOperand( MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, - 0); + 1); MIRBuilder.buildLoad(ValVReg, Addr, *MMO); } @@ -97,6 +99,8 @@ struct IncomingArgHandler : public CallLowering::ValueHandler { /// (it's an implicit-def of the BL). virtual void markPhysRegUsed(unsigned PhysReg) = 0; + bool isArgumentHandler() const override { return true; } + uint64_t StackUsed; }; @@ -129,31 +133,31 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler { : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB), AssignFnVarArg(AssignFnVarArg), StackSize(0) {} - unsigned getStackAddress(uint64_t Size, int64_t Offset, + Register getStackAddress(uint64_t Size, int64_t Offset, MachinePointerInfo &MPO) override { LLT p0 = LLT::pointer(0, 64); LLT s64 = LLT::scalar(64); - unsigned SPReg = MRI.createGenericVirtualRegister(p0); - MIRBuilder.buildCopy(SPReg, AArch64::SP); + Register SPReg = MRI.createGenericVirtualRegister(p0); + MIRBuilder.buildCopy(SPReg, Register(AArch64::SP)); - unsigned OffsetReg = MRI.createGenericVirtualRegister(s64); + Register OffsetReg = MRI.createGenericVirtualRegister(s64); MIRBuilder.buildConstant(OffsetReg, Offset); - unsigned AddrReg = MRI.createGenericVirtualRegister(p0); + Register AddrReg = MRI.createGenericVirtualRegister(p0); MIRBuilder.buildGEP(AddrReg, SPReg, OffsetReg); MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); return AddrReg; } - void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign &VA) override { MIB.addUse(PhysReg, RegState::Implicit); - unsigned ExtReg = extendRegister(ValVReg, VA); + Register ExtReg = extendRegister(ValVReg, VA); MIRBuilder.buildCopy(PhysReg, ExtReg); } - void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size, + void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) override { if (VA.getLocInfo() == CCValAssign::LocInfo::AExt) { Size = VA.getLocVT().getSizeInBits() / 8; @@ -162,7 +166,7 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler { .getReg(); } auto MMO = MIRBuilder.getMF().getMachineMemOperand( - MPO, MachineMemOperand::MOStore, Size, 0); + MPO, MachineMemOperand::MOStore, Size, 1); MIRBuilder.buildStore(ValVReg, Addr, *MMO); } @@ -188,8 +192,7 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler { void AArch64CallLowering::splitToValueTypes( const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs, - const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv, - const SplitArgTy &PerformArgSplit) const { + const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv) const { const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>(); LLVMContext &Ctx = OrigArg.Ty->getContext(); @@ -203,32 +206,31 @@ void AArch64CallLowering::splitToValueTypes( if (SplitVTs.size() == 1) { // No splitting to do, but we want to replace the original type (e.g. [1 x // double] -> double). - SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx), + SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx), OrigArg.Flags, OrigArg.IsFixed); return; } - unsigned FirstRegIdx = SplitArgs.size(); + // Create one ArgInfo for each virtual register in the original ArgInfo. + assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch"); + bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( OrigArg.Ty, CallConv, false); - for (auto SplitVT : SplitVTs) { - Type *SplitTy = SplitVT.getTypeForEVT(Ctx); - SplitArgs.push_back( - ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*SplitTy, DL)), - SplitTy, OrigArg.Flags, OrigArg.IsFixed}); + for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { + Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx); + SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.Flags, + OrigArg.IsFixed); if (NeedsRegBlock) SplitArgs.back().Flags.setInConsecutiveRegs(); } SplitArgs.back().Flags.setInConsecutiveRegsLast(); - - for (unsigned i = 0; i < Offsets.size(); ++i) - PerformArgSplit(SplitArgs[FirstRegIdx + i].Reg, Offsets[i] * 8); } bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, - ArrayRef<unsigned> VRegs) const { + ArrayRef<Register> VRegs, + Register SwiftErrorVReg) const { auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR); assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && "Return value without a vreg"); @@ -250,34 +252,101 @@ bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, "For each split Type there should be exactly one VReg."); SmallVector<ArgInfo, 8> SplitArgs; + CallingConv::ID CC = F.getCallingConv(); + for (unsigned i = 0; i < SplitEVTs.size(); ++i) { - // We zero-extend i1s to i8. - unsigned CurVReg = VRegs[i]; - if (MRI.getType(VRegs[i]).getSizeInBits() == 1) { - CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg) - ->getOperand(0) - .getReg(); + if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) > 1) { + LLVM_DEBUG(dbgs() << "Can't handle extended arg types which need split"); + return false; } + Register CurVReg = VRegs[i]; ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx)}; setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); - splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, F.getCallingConv(), - [&](unsigned Reg, uint64_t Offset) { - MIRBuilder.buildExtract(Reg, CurVReg, Offset); - }); + + // i1 is a special case because SDAG i1 true is naturally zero extended + // when widened using ANYEXT. We need to do it explicitly here. + if (MRI.getType(CurVReg).getSizeInBits() == 1) { + CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0); + } else { + // Some types will need extending as specified by the CC. + MVT NewVT = TLI.getRegisterTypeForCallingConv(Ctx, CC, SplitEVTs[i]); + if (EVT(NewVT) != SplitEVTs[i]) { + unsigned ExtendOp = TargetOpcode::G_ANYEXT; + if (F.getAttributes().hasAttribute(AttributeList::ReturnIndex, + Attribute::SExt)) + ExtendOp = TargetOpcode::G_SEXT; + else if (F.getAttributes().hasAttribute(AttributeList::ReturnIndex, + Attribute::ZExt)) + ExtendOp = TargetOpcode::G_ZEXT; + + LLT NewLLT(NewVT); + LLT OldLLT(MVT::getVT(CurArgInfo.Ty)); + CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Ctx); + // Instead of an extend, we might have a vector type which needs + // padding with more elements, e.g. <2 x half> -> <4 x half>. + if (NewVT.isVector()) { + if (OldLLT.isVector()) { + if (NewLLT.getNumElements() > OldLLT.getNumElements()) { + // We don't handle VA types which are not exactly twice the + // size, but can easily be done in future. + if (NewLLT.getNumElements() != OldLLT.getNumElements() * 2) { + LLVM_DEBUG(dbgs() << "Outgoing vector ret has too many elts"); + return false; + } + auto Undef = MIRBuilder.buildUndef({OldLLT}); + CurVReg = + MIRBuilder.buildMerge({NewLLT}, {CurVReg, Undef.getReg(0)}) + .getReg(0); + } else { + // Just do a vector extend. + CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}) + .getReg(0); + } + } else if (NewLLT.getNumElements() == 2) { + // We need to pad a <1 x S> type to <2 x S>. Since we don't have + // <1 x S> vector types in GISel we use a build_vector instead + // of a vector merge/concat. + auto Undef = MIRBuilder.buildUndef({OldLLT}); + CurVReg = + MIRBuilder + .buildBuildVector({NewLLT}, {CurVReg, Undef.getReg(0)}) + .getReg(0); + } else { + LLVM_DEBUG(dbgs() << "Could not handle ret ty"); + return false; + } + } else { + // A scalar extend. + CurVReg = + MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}).getReg(0); + } + } + } + if (CurVReg != CurArgInfo.Regs[0]) { + CurArgInfo.Regs[0] = CurVReg; + // Reset the arg flags after modifying CurVReg. + setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); + } + splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, CC); } OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFn, AssignFn); Success = handleAssignments(MIRBuilder, SplitArgs, Handler); } + if (SwiftErrorVReg) { + MIB.addUse(AArch64::X21, RegState::Implicit); + MIRBuilder.buildCopy(AArch64::X21, SwiftErrorVReg); + } + MIRBuilder.insertInstr(MIB); return Success; } -bool AArch64CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, - const Function &F, - ArrayRef<unsigned> VRegs) const { +bool AArch64CallLowering::lowerFormalArguments( + MachineIRBuilder &MIRBuilder, const Function &F, + ArrayRef<ArrayRef<Register>> VRegs) const { MachineFunction &MF = MIRBuilder.getMF(); MachineBasicBlock &MBB = MIRBuilder.getMBB(); MachineRegisterInfo &MRI = MF.getRegInfo(); @@ -288,26 +357,11 @@ bool AArch64CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, for (auto &Arg : F.args()) { if (DL.getTypeStoreSize(Arg.getType()) == 0) continue; + ArgInfo OrigArg{VRegs[i], Arg.getType()}; setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F); - bool Split = false; - LLT Ty = MRI.getType(VRegs[i]); - unsigned Dst = VRegs[i]; - - splitToValueTypes(OrigArg, SplitArgs, DL, MRI, F.getCallingConv(), - [&](unsigned Reg, uint64_t Offset) { - if (!Split) { - Split = true; - Dst = MRI.createGenericVirtualRegister(Ty); - MIRBuilder.buildUndef(Dst); - } - unsigned Tmp = MRI.createGenericVirtualRegister(Ty); - MIRBuilder.buildInsert(Tmp, Dst, Reg, Offset); - Dst = Tmp; - }); - if (Dst != VRegs[i]) - MIRBuilder.buildCopy(VRegs[i], Dst); + splitToValueTypes(OrigArg, SplitArgs, DL, MRI, F.getCallingConv()); ++i; } @@ -351,7 +405,8 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, const MachineOperand &Callee, const ArgInfo &OrigRet, - ArrayRef<ArgInfo> OrigArgs) const { + ArrayRef<ArgInfo> OrigArgs, + Register SwiftErrorVReg) const { MachineFunction &MF = MIRBuilder.getMF(); const Function &F = MF.getFunction(); MachineRegisterInfo &MRI = MF.getRegInfo(); @@ -359,10 +414,10 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, SmallVector<ArgInfo, 8> SplitArgs; for (auto &OrigArg : OrigArgs) { - splitToValueTypes(OrigArg, SplitArgs, DL, MRI, CallConv, - [&](unsigned Reg, uint64_t Offset) { - MIRBuilder.buildExtract(Reg, OrigArg.Reg, Offset); - }); + splitToValueTypes(OrigArg, SplitArgs, DL, MRI, CallConv); + // AAPCS requires that we zero-extend i1 to 8 bits by the caller. + if (OrigArg.Ty->isIntegerTy(1)) + SplitArgs.back().Flags.setZExt(); } // Find out which ABI gets to decide where things go. @@ -412,23 +467,19 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, // symmetry with the arugments, the physical register must be an // implicit-define of the call instruction. CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(F.getCallingConv()); - if (OrigRet.Reg) { + if (!OrigRet.Ty->isVoidTy()) { SplitArgs.clear(); - SmallVector<uint64_t, 8> RegOffsets; - SmallVector<unsigned, 8> SplitRegs; - splitToValueTypes(OrigRet, SplitArgs, DL, MRI, F.getCallingConv(), - [&](unsigned Reg, uint64_t Offset) { - RegOffsets.push_back(Offset); - SplitRegs.push_back(Reg); - }); + splitToValueTypes(OrigRet, SplitArgs, DL, MRI, F.getCallingConv()); CallReturnHandler Handler(MIRBuilder, MRI, MIB, RetAssignFn); if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) return false; + } - if (!RegOffsets.empty()) - MIRBuilder.buildSequence(OrigRet.Reg, SplitRegs, RegOffsets); + if (SwiftErrorVReg) { + MIB.addDef(AArch64::X21, RegState::Implicit); + MIRBuilder.buildCopy(SwiftErrorVReg, Register(AArch64::X21)); } CallSeqStart.addImm(Handler.StackSize).addImm(0); diff --git a/lib/Target/AArch64/AArch64CallLowering.h b/lib/Target/AArch64/AArch64CallLowering.h index 1c2bd6a4de5d..4f428f254537 100644 --- a/lib/Target/AArch64/AArch64CallLowering.h +++ b/lib/Target/AArch64/AArch64CallLowering.h @@ -1,9 +1,8 @@ //===- AArch64CallLowering.h - Call lowering --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -35,14 +34,24 @@ public: AArch64CallLowering(const AArch64TargetLowering &TLI); bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, - ArrayRef<unsigned> VRegs) const override; + ArrayRef<Register> VRegs, + Register SwiftErrorVReg) const override; bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, - ArrayRef<unsigned> VRegs) const override; + ArrayRef<ArrayRef<Register>> VRegs) const override; + + bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, + const MachineOperand &Callee, const ArgInfo &OrigRet, + ArrayRef<ArgInfo> OrigArgs, + Register SwiftErrorVReg) const override; bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, const MachineOperand &Callee, const ArgInfo &OrigRet, - ArrayRef<ArgInfo> OrigArgs) const override; + ArrayRef<ArgInfo> OrigArgs) const override { + return lowerCall(MIRBuilder, CallConv, Callee, OrigRet, OrigArgs, 0); + } + + bool supportSwiftError() const override { return true; } private: using RegHandler = std::function<void(MachineIRBuilder &, Type *, unsigned, @@ -51,13 +60,10 @@ private: using MemHandler = std::function<void(MachineIRBuilder &, int, CCValAssign &)>; - using SplitArgTy = std::function<void(unsigned, uint64_t)>; - void splitToValueTypes(const ArgInfo &OrigArgInfo, SmallVectorImpl<ArgInfo> &SplitArgs, const DataLayout &DL, MachineRegisterInfo &MRI, - CallingConv::ID CallConv, - const SplitArgTy &SplitArg) const; + CallingConv::ID CallConv) const; }; } // end namespace llvm diff --git a/lib/Target/AArch64/AArch64CallingConvention.cpp b/lib/Target/AArch64/AArch64CallingConvention.cpp new file mode 100644 index 000000000000..02538a187611 --- /dev/null +++ b/lib/Target/AArch64/AArch64CallingConvention.cpp @@ -0,0 +1,134 @@ +//=== AArch64CallingConvention.cpp - AArch64 CC impl ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the table-generated and custom routines for the AArch64 +// Calling Convention. +// +//===----------------------------------------------------------------------===// + +#include "AArch64CallingConvention.h" +#include "AArch64.h" +#include "AArch64InstrInfo.h" +#include "AArch64Subtarget.h" +#include "llvm/CodeGen/CallingConvLower.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/IR/CallingConv.h" +using namespace llvm; + +static const MCPhysReg XRegList[] = {AArch64::X0, AArch64::X1, AArch64::X2, + AArch64::X3, AArch64::X4, AArch64::X5, + AArch64::X6, AArch64::X7}; +static const MCPhysReg HRegList[] = {AArch64::H0, AArch64::H1, AArch64::H2, + AArch64::H3, AArch64::H4, AArch64::H5, + AArch64::H6, AArch64::H7}; +static const MCPhysReg SRegList[] = {AArch64::S0, AArch64::S1, AArch64::S2, + AArch64::S3, AArch64::S4, AArch64::S5, + AArch64::S6, AArch64::S7}; +static const MCPhysReg DRegList[] = {AArch64::D0, AArch64::D1, AArch64::D2, + AArch64::D3, AArch64::D4, AArch64::D5, + AArch64::D6, AArch64::D7}; +static const MCPhysReg QRegList[] = {AArch64::Q0, AArch64::Q1, AArch64::Q2, + AArch64::Q3, AArch64::Q4, AArch64::Q5, + AArch64::Q6, AArch64::Q7}; + +static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers, + MVT LocVT, ISD::ArgFlagsTy &ArgFlags, + CCState &State, unsigned SlotAlign) { + unsigned Size = LocVT.getSizeInBits() / 8; + unsigned StackAlign = + State.getMachineFunction().getDataLayout().getStackAlignment(); + unsigned Align = std::min(ArgFlags.getOrigAlign(), StackAlign); + + for (auto &It : PendingMembers) { + It.convertToMem(State.AllocateStack(Size, std::max(Align, SlotAlign))); + State.addLoc(It); + SlotAlign = 1; + } + + // All pending members have now been allocated + PendingMembers.clear(); + return true; +} + +/// The Darwin variadic PCS places anonymous arguments in 8-byte stack slots. An +/// [N x Ty] type must still be contiguous in memory though. +static bool CC_AArch64_Custom_Stack_Block( + unsigned &ValNo, MVT &ValVT, MVT &LocVT, CCValAssign::LocInfo &LocInfo, + ISD::ArgFlagsTy &ArgFlags, CCState &State) { + SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs(); + + // Add the argument to the list to be allocated once we know the size of the + // block. + PendingMembers.push_back( + CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); + + if (!ArgFlags.isInConsecutiveRegsLast()) + return true; + + return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, 8); +} + +/// Given an [N x Ty] block, it should be passed in a consecutive sequence of +/// registers. If no such sequence is available, mark the rest of the registers +/// of that type as used and place the argument on the stack. +static bool CC_AArch64_Custom_Block(unsigned &ValNo, MVT &ValVT, MVT &LocVT, + CCValAssign::LocInfo &LocInfo, + ISD::ArgFlagsTy &ArgFlags, CCState &State) { + // Try to allocate a contiguous block of registers, each of the correct + // size to hold one member. + ArrayRef<MCPhysReg> RegList; + if (LocVT.SimpleTy == MVT::i64) + RegList = XRegList; + else if (LocVT.SimpleTy == MVT::f16) + RegList = HRegList; + else if (LocVT.SimpleTy == MVT::f32 || LocVT.is32BitVector()) + RegList = SRegList; + else if (LocVT.SimpleTy == MVT::f64 || LocVT.is64BitVector()) + RegList = DRegList; + else if (LocVT.SimpleTy == MVT::f128 || LocVT.is128BitVector()) + RegList = QRegList; + else { + // Not an array we want to split up after all. + return false; + } + + SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs(); + + // Add the argument to the list to be allocated once we know the size of the + // block. + PendingMembers.push_back( + CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); + + if (!ArgFlags.isInConsecutiveRegsLast()) + return true; + + unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size()); + if (RegResult) { + for (auto &It : PendingMembers) { + It.convertToReg(RegResult); + State.addLoc(It); + ++RegResult; + } + PendingMembers.clear(); + return true; + } + + // Mark all regs in the class as unavailable + for (auto Reg : RegList) + State.AllocateReg(Reg); + + const AArch64Subtarget &Subtarget = static_cast<const AArch64Subtarget &>( + State.getMachineFunction().getSubtarget()); + unsigned SlotAlign = Subtarget.isTargetDarwin() ? 1 : 8; + + return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, SlotAlign); +} + +// TableGen provides definitions of the calling convention analysis entry +// points. +#include "AArch64GenCallingConv.inc" diff --git a/lib/Target/AArch64/AArch64CallingConvention.h b/lib/Target/AArch64/AArch64CallingConvention.h index 461c01318d4e..13cc0c583fd2 100644 --- a/lib/Target/AArch64/AArch64CallingConvention.h +++ b/lib/Target/AArch64/AArch64CallingConvention.h @@ -1,139 +1,45 @@ -//=== AArch64CallingConv.h - Custom Calling Convention Routines -*- C++ -*-===// +//=== AArch64CallingConvention.h - AArch64 CC entry points ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // -// This file contains the custom routines for the AArch64 Calling Convention -// that aren't done by tablegen. +// This file declares the entry points for AArch64 calling convention analysis. // //===----------------------------------------------------------------------===// #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64CALLINGCONVENTION_H #define LLVM_LIB_TARGET_AARCH64_AARCH64CALLINGCONVENTION_H -#include "AArch64.h" -#include "AArch64InstrInfo.h" -#include "AArch64Subtarget.h" #include "llvm/CodeGen/CallingConvLower.h" -#include "llvm/CodeGen/TargetInstrInfo.h" -#include "llvm/IR/CallingConv.h" - -namespace { -using namespace llvm; - -static const MCPhysReg XRegList[] = {AArch64::X0, AArch64::X1, AArch64::X2, - AArch64::X3, AArch64::X4, AArch64::X5, - AArch64::X6, AArch64::X7}; -static const MCPhysReg HRegList[] = {AArch64::H0, AArch64::H1, AArch64::H2, - AArch64::H3, AArch64::H4, AArch64::H5, - AArch64::H6, AArch64::H7}; -static const MCPhysReg SRegList[] = {AArch64::S0, AArch64::S1, AArch64::S2, - AArch64::S3, AArch64::S4, AArch64::S5, - AArch64::S6, AArch64::S7}; -static const MCPhysReg DRegList[] = {AArch64::D0, AArch64::D1, AArch64::D2, - AArch64::D3, AArch64::D4, AArch64::D5, - AArch64::D6, AArch64::D7}; -static const MCPhysReg QRegList[] = {AArch64::Q0, AArch64::Q1, AArch64::Q2, - AArch64::Q3, AArch64::Q4, AArch64::Q5, - AArch64::Q6, AArch64::Q7}; - -static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers, - MVT LocVT, ISD::ArgFlagsTy &ArgFlags, - CCState &State, unsigned SlotAlign) { - unsigned Size = LocVT.getSizeInBits() / 8; - unsigned StackAlign = - State.getMachineFunction().getDataLayout().getStackAlignment(); - unsigned Align = std::min(ArgFlags.getOrigAlign(), StackAlign); - - for (auto &It : PendingMembers) { - It.convertToMem(State.AllocateStack(Size, std::max(Align, SlotAlign))); - State.addLoc(It); - SlotAlign = 1; - } - - // All pending members have now been allocated - PendingMembers.clear(); - return true; -} - -/// The Darwin variadic PCS places anonymous arguments in 8-byte stack slots. An -/// [N x Ty] type must still be contiguous in memory though. -static bool CC_AArch64_Custom_Stack_Block( - unsigned &ValNo, MVT &ValVT, MVT &LocVT, CCValAssign::LocInfo &LocInfo, - ISD::ArgFlagsTy &ArgFlags, CCState &State) { - SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs(); - - // Add the argument to the list to be allocated once we know the size of the - // block. - PendingMembers.push_back( - CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); - - if (!ArgFlags.isInConsecutiveRegsLast()) - return true; - - return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, 8); -} - -/// Given an [N x Ty] block, it should be passed in a consecutive sequence of -/// registers. If no such sequence is available, mark the rest of the registers -/// of that type as used and place the argument on the stack. -static bool CC_AArch64_Custom_Block(unsigned &ValNo, MVT &ValVT, MVT &LocVT, - CCValAssign::LocInfo &LocInfo, - ISD::ArgFlagsTy &ArgFlags, CCState &State) { - // Try to allocate a contiguous block of registers, each of the correct - // size to hold one member. - ArrayRef<MCPhysReg> RegList; - if (LocVT.SimpleTy == MVT::i64) - RegList = XRegList; - else if (LocVT.SimpleTy == MVT::f16) - RegList = HRegList; - else if (LocVT.SimpleTy == MVT::f32 || LocVT.is32BitVector()) - RegList = SRegList; - else if (LocVT.SimpleTy == MVT::f64 || LocVT.is64BitVector()) - RegList = DRegList; - else if (LocVT.SimpleTy == MVT::f128 || LocVT.is128BitVector()) - RegList = QRegList; - else { - // Not an array we want to split up after all. - return false; - } - - SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs(); - - // Add the argument to the list to be allocated once we know the size of the - // block. - PendingMembers.push_back( - CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); - - if (!ArgFlags.isInConsecutiveRegsLast()) - return true; - - unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size()); - if (RegResult) { - for (auto &It : PendingMembers) { - It.convertToReg(RegResult); - State.addLoc(It); - ++RegResult; - } - PendingMembers.clear(); - return true; - } - - // Mark all regs in the class as unavailable - for (auto Reg : RegList) - State.AllocateReg(Reg); - - const AArch64Subtarget &Subtarget = static_cast<const AArch64Subtarget &>( - State.getMachineFunction().getSubtarget()); - unsigned SlotAlign = Subtarget.isTargetDarwin() ? 1 : 8; - - return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, SlotAlign); -} -} +namespace llvm { +bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + CCState &State); +bool CC_AArch64_DarwinPCS_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, + ISD::ArgFlagsTy ArgFlags, CCState &State); +bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, + ISD::ArgFlagsTy ArgFlags, CCState &State); +bool CC_AArch64_Win64_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, + ISD::ArgFlagsTy ArgFlags, CCState &State); +bool CC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, + ISD::ArgFlagsTy ArgFlags, CCState &State); +bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + CCState &State); +bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + CCState &State); +bool RetCC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, + ISD::ArgFlagsTy ArgFlags, CCState &State); +} // namespace llvm #endif diff --git a/lib/Target/AArch64/AArch64CallingConvention.td b/lib/Target/AArch64/AArch64CallingConvention.td index 5db941e9dac7..d969a9e1ab3a 100644 --- a/lib/Target/AArch64/AArch64CallingConvention.td +++ b/lib/Target/AArch64/AArch64CallingConvention.td @@ -1,9 +1,8 @@ //=- AArch64CallingConv.td - Calling Conventions for AArch64 -*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,6 +21,7 @@ class CCIfBigEndian<CCAction A> : // ARM AAPCS64 Calling Convention //===----------------------------------------------------------------------===// +let Entry = 1 in def CC_AArch64_AAPCS : CallingConv<[ CCIfType<[iPTR], CCBitConvertToType<i64>>, CCIfType<[v2f32], CCBitConvertToType<v2i32>>, @@ -34,7 +34,23 @@ def CC_AArch64_AAPCS : CallingConv<[ CCIfBigEndian<CCIfType<[v2i64, v2f64, v4i32, v4f32, v8i16, v8f16, v16i8], CCBitConvertToType<f128>>>, - // An SRet is passed in X8, not X0 like a normal pointer parameter. + // In AAPCS, an SRet is passed in X8, not X0 like a normal pointer parameter. + // However, on windows, in some circumstances, the SRet is passed in X0 or X1 + // instead. The presence of the inreg attribute indicates that SRet is + // passed in the alternative register (X0 or X1), not X8: + // - X0 for non-instance methods. + // - X1 for instance methods. + + // The "sret" attribute identifies indirect returns. + // The "inreg" attribute identifies non-aggregate types. + // The position of the "sret" attribute identifies instance/non-instance + // methods. + // "sret" on argument 0 means non-instance methods. + // "sret" on argument 1 means instance methods. + + CCIfInReg<CCIfType<[i64], + CCIfSRet<CCIfType<[i64], CCAssignToRegWithShadow<[X0, X1], [W0, W1]>>>>>, + CCIfSRet<CCIfType<[i64], CCAssignToRegWithShadow<[X8], [W8]>>>, // Put ByVal arguments directly on the stack. Minimum size and alignment of a @@ -89,6 +105,7 @@ def CC_AArch64_AAPCS : CallingConv<[ CCAssignToStack<16, 16>> ]>; +let Entry = 1 in def RetCC_AArch64_AAPCS : CallingConv<[ CCIfType<[iPTR], CCBitConvertToType<i64>>, CCIfType<[v2f32], CCBitConvertToType<v2i32>>, @@ -122,6 +139,7 @@ def RetCC_AArch64_AAPCS : CallingConv<[ ]>; // Vararg functions on windows pass floats in integer registers +let Entry = 1 in def CC_AArch64_Win64_VarArg : CallingConv<[ CCIfType<[f16, f32], CCPromoteToType<f64>>, CCIfType<[f64], CCBitConvertToType<i64>>, @@ -133,6 +151,7 @@ def CC_AArch64_Win64_VarArg : CallingConv<[ // from the standard one at this level: // + i128s (i.e. split i64s) don't need even registers. // + Stack slots are sized as needed rather than being at least 64-bit. +let Entry = 1 in def CC_AArch64_DarwinPCS : CallingConv<[ CCIfType<[iPTR], CCBitConvertToType<i64>>, CCIfType<[v2f32], CCBitConvertToType<v2i32>>, @@ -189,6 +208,7 @@ def CC_AArch64_DarwinPCS : CallingConv<[ CCAssignToStack<16, 16>> ]>; +let Entry = 1 in def CC_AArch64_DarwinPCS_VarArg : CallingConv<[ CCIfType<[iPTR], CCBitConvertToType<i64>>, CCIfType<[v2f32], CCBitConvertToType<v2i32>>, @@ -213,6 +233,7 @@ def CC_AArch64_DarwinPCS_VarArg : CallingConv<[ // in register and the remaining arguments on stack. We allow 32bit stack slots, // so that WebKit can write partial values in the stack and define the other // 32bit quantity as undef. +let Entry = 1 in def CC_AArch64_WebKit_JS : CallingConv<[ // Handle i1, i8, i16, i32, and i64 passing in register X0 (W0). CCIfType<[i1, i8, i16], CCPromoteToType<i32>>, @@ -224,6 +245,7 @@ def CC_AArch64_WebKit_JS : CallingConv<[ CCIfType<[i64, f64], CCAssignToStack<8, 8>> ]>; +let Entry = 1 in def RetCC_AArch64_WebKit_JS : CallingConv<[ CCIfType<[i32], CCAssignToRegWithShadow<[W0, W1, W2, W3, W4, W5, W6, W7], [X0, X1, X2, X3, X4, X5, X6, X7]>>, @@ -257,6 +279,7 @@ def RetCC_AArch64_WebKit_JS : CallingConv<[ // The AArch64 register mapping is under the heading "The ARMv8/AArch64 ABI // register mapping". +let Entry = 1 in def CC_AArch64_GHC : CallingConv<[ CCIfType<[iPTR], CCBitConvertToType<i64>>, diff --git a/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp b/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp index b88fba4452a1..688bd1b28e85 100644 --- a/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp +++ b/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp @@ -1,9 +1,8 @@ //===-- AArch64CleanupLocalDynamicTLSPass.cpp ---------------------*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64CollectLOH.cpp b/lib/Target/AArch64/AArch64CollectLOH.cpp index 720323f81d29..9f324b433209 100644 --- a/lib/Target/AArch64/AArch64CollectLOH.cpp +++ b/lib/Target/AArch64/AArch64CollectLOH.cpp @@ -1,9 +1,8 @@ //===---------- AArch64CollectLOH.cpp - AArch64 collect LOH pass --*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64CompressJumpTables.cpp b/lib/Target/AArch64/AArch64CompressJumpTables.cpp index 0924a27e2586..48dab79b32d3 100644 --- a/lib/Target/AArch64/AArch64CompressJumpTables.cpp +++ b/lib/Target/AArch64/AArch64CompressJumpTables.cpp @@ -1,9 +1,8 @@ //==-- AArch64CompressJumpTables.cpp - Compress jump tables for AArch64 --====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // This pass looks at the basic blocks each jump-table refers to and works out // whether they can be emitted in a compressed form (with 8 or 16-bit @@ -108,6 +107,7 @@ bool AArch64CompressJumpTables::compressJumpTable(MachineInstr &MI, MinBlock = Block; } } + assert(MinBlock && "Failed to find minimum offset block"); // The ADR instruction needed to calculate the address of the first reachable // basic block can address +/-1MB. @@ -141,7 +141,7 @@ bool AArch64CompressJumpTables::runOnMachineFunction(MachineFunction &MFIn) { const auto &ST = MF->getSubtarget<AArch64Subtarget>(); TII = ST.getInstrInfo(); - if (ST.force32BitJumpTables() && !MF->getFunction().optForMinSize()) + if (ST.force32BitJumpTables() && !MF->getFunction().hasMinSize()) return false; scanFunction(); diff --git a/lib/Target/AArch64/AArch64CondBrTuning.cpp b/lib/Target/AArch64/AArch64CondBrTuning.cpp index 5ae787409ae8..453132e09669 100644 --- a/lib/Target/AArch64/AArch64CondBrTuning.cpp +++ b/lib/Target/AArch64/AArch64CondBrTuning.cpp @@ -1,9 +1,8 @@ //===-- AArch64CondBrTuning.cpp --- Conditional branch tuning for AArch64 -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp index 5064762b9f77..a6efb115ed44 100644 --- a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp +++ b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp @@ -1,9 +1,8 @@ //=- AArch64ConditionOptimizer.cpp - Remove useless comparisons for AArch64 -=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64ConditionalCompares.cpp b/lib/Target/AArch64/AArch64ConditionalCompares.cpp index 8176b6fb269d..2cfbcc592d6a 100644 --- a/lib/Target/AArch64/AArch64ConditionalCompares.cpp +++ b/lib/Target/AArch64/AArch64ConditionalCompares.cpp @@ -1,9 +1,8 @@ //===-- AArch64ConditionalCompares.cpp --- CCMP formation for AArch64 -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -941,7 +940,7 @@ bool AArch64ConditionalCompares::runOnMachineFunction(MachineFunction &MF) { MBPI = &getAnalysis<MachineBranchProbabilityInfo>(); Traces = &getAnalysis<MachineTraceMetrics>(); MinInstr = nullptr; - MinSize = MF.getFunction().optForMinSize(); + MinSize = MF.getFunction().hasMinSize(); bool Changed = false; CmpConv.runOnMachineFunction(MF, MBPI); diff --git a/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp b/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp index 2ba10d25e939..a43077cb88ec 100644 --- a/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp +++ b/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp @@ -1,9 +1,8 @@ //==-- AArch64DeadRegisterDefinitions.cpp - Replace dead defs w/ zero reg --==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file When allowed by the instruction, replace a dead definition of a GPR @@ -55,8 +54,6 @@ public: AU.setPreservesCFG(); MachineFunctionPass::getAnalysisUsage(AU); } - - bool shouldSkip(const MachineInstr &MI, const MachineFunction &MF) const; }; char AArch64DeadRegisterDefinitions::ID = 0; } // end anonymous namespace @@ -71,60 +68,48 @@ static bool usesFrameIndex(const MachineInstr &MI) { return false; } -bool -AArch64DeadRegisterDefinitions::shouldSkip(const MachineInstr &MI, - const MachineFunction &MF) const { - if (!MF.getSubtarget<AArch64Subtarget>().hasLSE()) - return false; - -#define CASE_AARCH64_ATOMIC_(PREFIX) \ - case AArch64::PREFIX##X: \ - case AArch64::PREFIX##W: \ - case AArch64::PREFIX##H: \ - case AArch64::PREFIX##B - - for (const MachineMemOperand *MMO : MI.memoperands()) { - if (MMO->isAtomic()) { - unsigned Opcode = MI.getOpcode(); - switch (Opcode) { - default: - return false; - break; - - CASE_AARCH64_ATOMIC_(LDADDA): - CASE_AARCH64_ATOMIC_(LDADDAL): - - CASE_AARCH64_ATOMIC_(LDCLRA): - CASE_AARCH64_ATOMIC_(LDCLRAL): - - CASE_AARCH64_ATOMIC_(LDEORA): - CASE_AARCH64_ATOMIC_(LDEORAL): - - CASE_AARCH64_ATOMIC_(LDSETA): - CASE_AARCH64_ATOMIC_(LDSETAL): - - CASE_AARCH64_ATOMIC_(LDSMAXA): - CASE_AARCH64_ATOMIC_(LDSMAXAL): - - CASE_AARCH64_ATOMIC_(LDSMINA): - CASE_AARCH64_ATOMIC_(LDSMINAL): - - CASE_AARCH64_ATOMIC_(LDUMAXA): - CASE_AARCH64_ATOMIC_(LDUMAXAL): - - CASE_AARCH64_ATOMIC_(LDUMINA): - CASE_AARCH64_ATOMIC_(LDUMINAL): - - CASE_AARCH64_ATOMIC_(SWPA): - CASE_AARCH64_ATOMIC_(SWPAL): - return true; - break; - } - } +// Instructions that lose their 'read' operation for a subesquent fence acquire +// (DMB LD) once the zero register is used. +// +// WARNING: The aquire variants of the instructions are also affected, but they +// are split out into `atomicBarrierDroppedOnZero()` to support annotations on +// assembly. +static bool atomicReadDroppedOnZero(unsigned Opcode) { + switch (Opcode) { + case AArch64::LDADDB: case AArch64::LDADDH: + case AArch64::LDADDW: case AArch64::LDADDX: + case AArch64::LDADDLB: case AArch64::LDADDLH: + case AArch64::LDADDLW: case AArch64::LDADDLX: + case AArch64::LDCLRB: case AArch64::LDCLRH: + case AArch64::LDCLRW: case AArch64::LDCLRX: + case AArch64::LDCLRLB: case AArch64::LDCLRLH: + case AArch64::LDCLRLW: case AArch64::LDCLRLX: + case AArch64::LDEORB: case AArch64::LDEORH: + case AArch64::LDEORW: case AArch64::LDEORX: + case AArch64::LDEORLB: case AArch64::LDEORLH: + case AArch64::LDEORLW: case AArch64::LDEORLX: + case AArch64::LDSETB: case AArch64::LDSETH: + case AArch64::LDSETW: case AArch64::LDSETX: + case AArch64::LDSETLB: case AArch64::LDSETLH: + case AArch64::LDSETLW: case AArch64::LDSETLX: + case AArch64::LDSMAXB: case AArch64::LDSMAXH: + case AArch64::LDSMAXW: case AArch64::LDSMAXX: + case AArch64::LDSMAXLB: case AArch64::LDSMAXLH: + case AArch64::LDSMAXLW: case AArch64::LDSMAXLX: + case AArch64::LDSMINB: case AArch64::LDSMINH: + case AArch64::LDSMINW: case AArch64::LDSMINX: + case AArch64::LDSMINLB: case AArch64::LDSMINLH: + case AArch64::LDSMINLW: case AArch64::LDSMINLX: + case AArch64::LDUMAXB: case AArch64::LDUMAXH: + case AArch64::LDUMAXW: case AArch64::LDUMAXX: + case AArch64::LDUMAXLB: case AArch64::LDUMAXLH: + case AArch64::LDUMAXLW: case AArch64::LDUMAXLX: + case AArch64::LDUMINB: case AArch64::LDUMINH: + case AArch64::LDUMINW: case AArch64::LDUMINX: + case AArch64::LDUMINLB: case AArch64::LDUMINLH: + case AArch64::LDUMINLW: case AArch64::LDUMINLX: + return true; } - -#undef CASE_AARCH64_ATOMIC_ - return false; } @@ -148,9 +133,8 @@ void AArch64DeadRegisterDefinitions::processMachineBasicBlock( continue; } - if (shouldSkip(MI, MF)) { - LLVM_DEBUG(dbgs() << " Ignoring, Atomic instruction with acquire " - "semantics using WZR/XZR\n"); + if (atomicBarrierDroppedOnZero(MI.getOpcode()) || atomicReadDroppedOnZero(MI.getOpcode())) { + LLVM_DEBUG(dbgs() << " Ignoring, semantics change with xzr/wzr.\n"); continue; } diff --git a/lib/Target/AArch64/AArch64ExpandImm.cpp b/lib/Target/AArch64/AArch64ExpandImm.cpp new file mode 100644 index 000000000000..c764af80eb86 --- /dev/null +++ b/lib/Target/AArch64/AArch64ExpandImm.cpp @@ -0,0 +1,411 @@ +//===- AArch64ExpandImm.h - AArch64 Immediate Expansion -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the AArch64ExpandImm stuff. +// +//===----------------------------------------------------------------------===// + +#include "AArch64.h" +#include "AArch64ExpandImm.h" +#include "MCTargetDesc/AArch64AddressingModes.h" + +namespace llvm { + +namespace AArch64_IMM { + +/// Helper function which extracts the specified 16-bit chunk from a +/// 64-bit value. +static uint64_t getChunk(uint64_t Imm, unsigned ChunkIdx) { + assert(ChunkIdx < 4 && "Out of range chunk index specified!"); + + return (Imm >> (ChunkIdx * 16)) & 0xFFFF; +} + +/// Check whether the given 16-bit chunk replicated to full 64-bit width +/// can be materialized with an ORR instruction. +static bool canUseOrr(uint64_t Chunk, uint64_t &Encoding) { + Chunk = (Chunk << 48) | (Chunk << 32) | (Chunk << 16) | Chunk; + + return AArch64_AM::processLogicalImmediate(Chunk, 64, Encoding); +} + +/// Check for identical 16-bit chunks within the constant and if so +/// materialize them with a single ORR instruction. The remaining one or two +/// 16-bit chunks will be materialized with MOVK instructions. +/// +/// This allows us to materialize constants like |A|B|A|A| or |A|B|C|A| (order +/// of the chunks doesn't matter), assuming |A|A|A|A| can be materialized with +/// an ORR instruction. +static bool tryToreplicateChunks(uint64_t UImm, + SmallVectorImpl<ImmInsnModel> &Insn) { + using CountMap = DenseMap<uint64_t, unsigned>; + + CountMap Counts; + + // Scan the constant and count how often every chunk occurs. + for (unsigned Idx = 0; Idx < 4; ++Idx) + ++Counts[getChunk(UImm, Idx)]; + + // Traverse the chunks to find one which occurs more than once. + for (CountMap::const_iterator Chunk = Counts.begin(), End = Counts.end(); + Chunk != End; ++Chunk) { + const uint64_t ChunkVal = Chunk->first; + const unsigned Count = Chunk->second; + + uint64_t Encoding = 0; + + // We are looking for chunks which have two or three instances and can be + // materialized with an ORR instruction. + if ((Count != 2 && Count != 3) || !canUseOrr(ChunkVal, Encoding)) + continue; + + const bool CountThree = Count == 3; + + Insn.push_back({ AArch64::ORRXri, 0, Encoding }); + + unsigned ShiftAmt = 0; + uint64_t Imm16 = 0; + // Find the first chunk not materialized with the ORR instruction. + for (; ShiftAmt < 64; ShiftAmt += 16) { + Imm16 = (UImm >> ShiftAmt) & 0xFFFF; + + if (Imm16 != ChunkVal) + break; + } + + // Create the first MOVK instruction. + Insn.push_back({ AArch64::MOVKXi, Imm16, + AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt) }); + + // In case we have three instances the whole constant is now materialized + // and we can exit. + if (CountThree) + return true; + + // Find the remaining chunk which needs to be materialized. + for (ShiftAmt += 16; ShiftAmt < 64; ShiftAmt += 16) { + Imm16 = (UImm >> ShiftAmt) & 0xFFFF; + + if (Imm16 != ChunkVal) + break; + } + Insn.push_back({ AArch64::MOVKXi, Imm16, + AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt) }); + return true; + } + + return false; +} + +/// Check whether this chunk matches the pattern '1...0...'. This pattern +/// starts a contiguous sequence of ones if we look at the bits from the LSB +/// towards the MSB. +static bool isStartChunk(uint64_t Chunk) { + if (Chunk == 0 || Chunk == std::numeric_limits<uint64_t>::max()) + return false; + + return isMask_64(~Chunk); +} + +/// Check whether this chunk matches the pattern '0...1...' This pattern +/// ends a contiguous sequence of ones if we look at the bits from the LSB +/// towards the MSB. +static bool isEndChunk(uint64_t Chunk) { + if (Chunk == 0 || Chunk == std::numeric_limits<uint64_t>::max()) + return false; + + return isMask_64(Chunk); +} + +/// Clear or set all bits in the chunk at the given index. +static uint64_t updateImm(uint64_t Imm, unsigned Idx, bool Clear) { + const uint64_t Mask = 0xFFFF; + + if (Clear) + // Clear chunk in the immediate. + Imm &= ~(Mask << (Idx * 16)); + else + // Set all bits in the immediate for the particular chunk. + Imm |= Mask << (Idx * 16); + + return Imm; +} + +/// Check whether the constant contains a sequence of contiguous ones, +/// which might be interrupted by one or two chunks. If so, materialize the +/// sequence of contiguous ones with an ORR instruction. +/// Materialize the chunks which are either interrupting the sequence or outside +/// of the sequence with a MOVK instruction. +/// +/// Assuming S is a chunk which starts the sequence (1...0...), E is a chunk +/// which ends the sequence (0...1...). Then we are looking for constants which +/// contain at least one S and E chunk. +/// E.g. |E|A|B|S|, |A|E|B|S| or |A|B|E|S|. +/// +/// We are also looking for constants like |S|A|B|E| where the contiguous +/// sequence of ones wraps around the MSB into the LSB. +static bool trySequenceOfOnes(uint64_t UImm, + SmallVectorImpl<ImmInsnModel> &Insn) { + const int NotSet = -1; + const uint64_t Mask = 0xFFFF; + + int StartIdx = NotSet; + int EndIdx = NotSet; + // Try to find the chunks which start/end a contiguous sequence of ones. + for (int Idx = 0; Idx < 4; ++Idx) { + int64_t Chunk = getChunk(UImm, Idx); + // Sign extend the 16-bit chunk to 64-bit. + Chunk = (Chunk << 48) >> 48; + + if (isStartChunk(Chunk)) + StartIdx = Idx; + else if (isEndChunk(Chunk)) + EndIdx = Idx; + } + + // Early exit in case we can't find a start/end chunk. + if (StartIdx == NotSet || EndIdx == NotSet) + return false; + + // Outside of the contiguous sequence of ones everything needs to be zero. + uint64_t Outside = 0; + // Chunks between the start and end chunk need to have all their bits set. + uint64_t Inside = Mask; + + // If our contiguous sequence of ones wraps around from the MSB into the LSB, + // just swap indices and pretend we are materializing a contiguous sequence + // of zeros surrounded by a contiguous sequence of ones. + if (StartIdx > EndIdx) { + std::swap(StartIdx, EndIdx); + std::swap(Outside, Inside); + } + + uint64_t OrrImm = UImm; + int FirstMovkIdx = NotSet; + int SecondMovkIdx = NotSet; + + // Find out which chunks we need to patch up to obtain a contiguous sequence + // of ones. + for (int Idx = 0; Idx < 4; ++Idx) { + const uint64_t Chunk = getChunk(UImm, Idx); + + // Check whether we are looking at a chunk which is not part of the + // contiguous sequence of ones. + if ((Idx < StartIdx || EndIdx < Idx) && Chunk != Outside) { + OrrImm = updateImm(OrrImm, Idx, Outside == 0); + + // Remember the index we need to patch. + if (FirstMovkIdx == NotSet) + FirstMovkIdx = Idx; + else + SecondMovkIdx = Idx; + + // Check whether we are looking a chunk which is part of the contiguous + // sequence of ones. + } else if (Idx > StartIdx && Idx < EndIdx && Chunk != Inside) { + OrrImm = updateImm(OrrImm, Idx, Inside != Mask); + + // Remember the index we need to patch. + if (FirstMovkIdx == NotSet) + FirstMovkIdx = Idx; + else + SecondMovkIdx = Idx; + } + } + assert(FirstMovkIdx != NotSet && "Constant materializable with single ORR!"); + + // Create the ORR-immediate instruction. + uint64_t Encoding = 0; + AArch64_AM::processLogicalImmediate(OrrImm, 64, Encoding); + Insn.push_back({ AArch64::ORRXri, 0, Encoding }); + + const bool SingleMovk = SecondMovkIdx == NotSet; + Insn.push_back({ AArch64::MOVKXi, getChunk(UImm, FirstMovkIdx), + AArch64_AM::getShifterImm(AArch64_AM::LSL, + FirstMovkIdx * 16) }); + + // Early exit in case we only need to emit a single MOVK instruction. + if (SingleMovk) + return true; + + // Create the second MOVK instruction. + Insn.push_back({ AArch64::MOVKXi, getChunk(UImm, SecondMovkIdx), + AArch64_AM::getShifterImm(AArch64_AM::LSL, + SecondMovkIdx * 16) }); + + return true; +} + +/// \brief Expand a MOVi32imm or MOVi64imm pseudo instruction to a +/// MOVZ or MOVN of width BitSize followed by up to 3 MOVK instructions. +static inline void expandMOVImmSimple(uint64_t Imm, unsigned BitSize, + unsigned OneChunks, unsigned ZeroChunks, + SmallVectorImpl<ImmInsnModel> &Insn) { + const unsigned Mask = 0xFFFF; + + // Use a MOVZ or MOVN instruction to set the high bits, followed by one or + // more MOVK instructions to insert additional 16-bit portions into the + // lower bits. + bool isNeg = false; + + // Use MOVN to materialize the high bits if we have more all one chunks + // than all zero chunks. + if (OneChunks > ZeroChunks) { + isNeg = true; + Imm = ~Imm; + } + + unsigned FirstOpc; + if (BitSize == 32) { + Imm &= (1LL << 32) - 1; + FirstOpc = (isNeg ? AArch64::MOVNWi : AArch64::MOVZWi); + } else { + FirstOpc = (isNeg ? AArch64::MOVNXi : AArch64::MOVZXi); + } + unsigned Shift = 0; // LSL amount for high bits with MOVZ/MOVN + unsigned LastShift = 0; // LSL amount for last MOVK + if (Imm != 0) { + unsigned LZ = countLeadingZeros(Imm); + unsigned TZ = countTrailingZeros(Imm); + Shift = (TZ / 16) * 16; + LastShift = ((63 - LZ) / 16) * 16; + } + unsigned Imm16 = (Imm >> Shift) & Mask; + + Insn.push_back({ FirstOpc, Imm16, + AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift) }); + + if (Shift == LastShift) + return; + + // If a MOVN was used for the high bits of a negative value, flip the rest + // of the bits back for use with MOVK. + if (isNeg) + Imm = ~Imm; + + unsigned Opc = (BitSize == 32 ? AArch64::MOVKWi : AArch64::MOVKXi); + while (Shift < LastShift) { + Shift += 16; + Imm16 = (Imm >> Shift) & Mask; + if (Imm16 == (isNeg ? Mask : 0)) + continue; // This 16-bit portion is already set correctly. + + Insn.push_back({ Opc, Imm16, + AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift) }); + } +} + +/// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more +/// real move-immediate instructions to synthesize the immediate. +void expandMOVImm(uint64_t Imm, unsigned BitSize, + SmallVectorImpl<ImmInsnModel> &Insn) { + const unsigned Mask = 0xFFFF; + + // Scan the immediate and count the number of 16-bit chunks which are either + // all ones or all zeros. + unsigned OneChunks = 0; + unsigned ZeroChunks = 0; + for (unsigned Shift = 0; Shift < BitSize; Shift += 16) { + const unsigned Chunk = (Imm >> Shift) & Mask; + if (Chunk == Mask) + OneChunks++; + else if (Chunk == 0) + ZeroChunks++; + } + + // Prefer MOVZ/MOVN over ORR because of the rules for the "mov" alias. + if ((BitSize / 16) - OneChunks <= 1 || (BitSize / 16) - ZeroChunks <= 1) { + expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); + return; + } + + // Try a single ORR. + uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize); + uint64_t Encoding; + if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) { + unsigned Opc = (BitSize == 32 ? AArch64::ORRWri : AArch64::ORRXri); + Insn.push_back({ Opc, 0, Encoding }); + return; + } + + // One to up three instruction sequences. + // + // Prefer MOVZ/MOVN followed by MOVK; it's more readable, and possibly the + // fastest sequence with fast literal generation. + if (OneChunks >= (BitSize / 16) - 2 || ZeroChunks >= (BitSize / 16) - 2) { + expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); + return; + } + + assert(BitSize == 64 && "All 32-bit immediates can be expanded with a" + "MOVZ/MOVK pair"); + + // Try other two-instruction sequences. + + // 64-bit ORR followed by MOVK. + // We try to construct the ORR immediate in three different ways: either we + // zero out the chunk which will be replaced, we fill the chunk which will + // be replaced with ones, or we take the bit pattern from the other half of + // the 64-bit immediate. This is comprehensive because of the way ORR + // immediates are constructed. + for (unsigned Shift = 0; Shift < BitSize; Shift += 16) { + uint64_t ShiftedMask = (0xFFFFULL << Shift); + uint64_t ZeroChunk = UImm & ~ShiftedMask; + uint64_t OneChunk = UImm | ShiftedMask; + uint64_t RotatedImm = (UImm << 32) | (UImm >> 32); + uint64_t ReplicateChunk = ZeroChunk | (RotatedImm & ShiftedMask); + if (AArch64_AM::processLogicalImmediate(ZeroChunk, BitSize, Encoding) || + AArch64_AM::processLogicalImmediate(OneChunk, BitSize, Encoding) || + AArch64_AM::processLogicalImmediate(ReplicateChunk, BitSize, + Encoding)) { + // Create the ORR-immediate instruction. + Insn.push_back({ AArch64::ORRXri, 0, Encoding }); + + // Create the MOVK instruction. + const unsigned Imm16 = getChunk(UImm, Shift / 16); + Insn.push_back({ AArch64::MOVKXi, Imm16, + AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift) }); + return; + } + } + + // FIXME: Add more two-instruction sequences. + + // Three instruction sequences. + // + // Prefer MOVZ/MOVN followed by two MOVK; it's more readable, and possibly + // the fastest sequence with fast literal generation. (If neither MOVK is + // part of a fast literal generation pair, it could be slower than the + // four-instruction sequence, but we won't worry about that for now.) + if (OneChunks || ZeroChunks) { + expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); + return; + } + + // Check for identical 16-bit chunks within the constant and if so materialize + // them with a single ORR instruction. The remaining one or two 16-bit chunks + // will be materialized with MOVK instructions. + if (BitSize == 64 && tryToreplicateChunks(UImm, Insn)) + return; + + // Check whether the constant contains a sequence of contiguous ones, which + // might be interrupted by one or two chunks. If so, materialize the sequence + // of contiguous ones with an ORR instruction. Materialize the chunks which + // are either interrupting the sequence or outside of the sequence with a + // MOVK instruction. + if (BitSize == 64 && trySequenceOfOnes(UImm, Insn)) + return; + + // We found no possible two or three instruction sequence; use the general + // four-instruction sequence. + expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); +} + +} // end namespace AArch64_AM + +} // end namespace llvm diff --git a/lib/Target/AArch64/AArch64ExpandImm.h b/lib/Target/AArch64/AArch64ExpandImm.h new file mode 100644 index 000000000000..42c97d2c3e9b --- /dev/null +++ b/lib/Target/AArch64/AArch64ExpandImm.h @@ -0,0 +1,35 @@ +//===- AArch64ExpandImm.h - AArch64 Immediate Expansion ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the AArch64 immediate expansion stuff. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64EXPANDIMM_H +#define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64EXPANDIMM_H + +#include "llvm/ADT/SmallVector.h" + +namespace llvm { + +namespace AArch64_IMM { + +struct ImmInsnModel { + unsigned Opcode; + uint64_t Op1; + uint64_t Op2; +}; + +void expandMOVImm(uint64_t Imm, unsigned BitSize, + SmallVectorImpl<ImmInsnModel> &Insn); + +} // end namespace AArch64_IMM + +} // end namespace llvm + +#endif diff --git a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp index f7190d58fbf9..210c10eb1842 100644 --- a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp +++ b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp @@ -1,9 +1,8 @@ //===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,7 +13,9 @@ // //===----------------------------------------------------------------------===// +#include "AArch64ExpandImm.h" #include "AArch64InstrInfo.h" +#include "AArch64MachineFunctionInfo.h" #include "AArch64Subtarget.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "Utils/AArch64BaseInfo.h" @@ -66,11 +67,6 @@ private: MachineBasicBlock::iterator &NextMBBI); bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned BitSize); - bool expandMOVImmSimple(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - unsigned BitSize, - unsigned OneChunks, - unsigned ZeroChunks); bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp, unsigned StlrOp, unsigned CmpOp, @@ -79,6 +75,9 @@ private: bool expandCMP_SWAP_128(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NextMBBI); + bool expandSetTagLoop(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); }; } // end anonymous namespace @@ -104,279 +103,6 @@ static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI, } } -/// Helper function which extracts the specified 16-bit chunk from a -/// 64-bit value. -static uint64_t getChunk(uint64_t Imm, unsigned ChunkIdx) { - assert(ChunkIdx < 4 && "Out of range chunk index specified!"); - - return (Imm >> (ChunkIdx * 16)) & 0xFFFF; -} - -/// Check whether the given 16-bit chunk replicated to full 64-bit width -/// can be materialized with an ORR instruction. -static bool canUseOrr(uint64_t Chunk, uint64_t &Encoding) { - Chunk = (Chunk << 48) | (Chunk << 32) | (Chunk << 16) | Chunk; - - return AArch64_AM::processLogicalImmediate(Chunk, 64, Encoding); -} - -/// Check for identical 16-bit chunks within the constant and if so -/// materialize them with a single ORR instruction. The remaining one or two -/// 16-bit chunks will be materialized with MOVK instructions. -/// -/// This allows us to materialize constants like |A|B|A|A| or |A|B|C|A| (order -/// of the chunks doesn't matter), assuming |A|A|A|A| can be materialized with -/// an ORR instruction. -static bool tryToreplicateChunks(uint64_t UImm, MachineInstr &MI, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator &MBBI, - const AArch64InstrInfo *TII) { - using CountMap = DenseMap<uint64_t, unsigned>; - - CountMap Counts; - - // Scan the constant and count how often every chunk occurs. - for (unsigned Idx = 0; Idx < 4; ++Idx) - ++Counts[getChunk(UImm, Idx)]; - - // Traverse the chunks to find one which occurs more than once. - for (CountMap::const_iterator Chunk = Counts.begin(), End = Counts.end(); - Chunk != End; ++Chunk) { - const uint64_t ChunkVal = Chunk->first; - const unsigned Count = Chunk->second; - - uint64_t Encoding = 0; - - // We are looking for chunks which have two or three instances and can be - // materialized with an ORR instruction. - if ((Count != 2 && Count != 3) || !canUseOrr(ChunkVal, Encoding)) - continue; - - const bool CountThree = Count == 3; - // Create the ORR-immediate instruction. - MachineInstrBuilder MIB = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXri)) - .add(MI.getOperand(0)) - .addReg(AArch64::XZR) - .addImm(Encoding); - - const unsigned DstReg = MI.getOperand(0).getReg(); - const bool DstIsDead = MI.getOperand(0).isDead(); - - unsigned ShiftAmt = 0; - uint64_t Imm16 = 0; - // Find the first chunk not materialized with the ORR instruction. - for (; ShiftAmt < 64; ShiftAmt += 16) { - Imm16 = (UImm >> ShiftAmt) & 0xFFFF; - - if (Imm16 != ChunkVal) - break; - } - - // Create the first MOVK instruction. - MachineInstrBuilder MIB1 = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi)) - .addReg(DstReg, - RegState::Define | getDeadRegState(DstIsDead && CountThree)) - .addReg(DstReg) - .addImm(Imm16) - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt)); - - // In case we have three instances the whole constant is now materialized - // and we can exit. - if (CountThree) { - transferImpOps(MI, MIB, MIB1); - MI.eraseFromParent(); - return true; - } - - // Find the remaining chunk which needs to be materialized. - for (ShiftAmt += 16; ShiftAmt < 64; ShiftAmt += 16) { - Imm16 = (UImm >> ShiftAmt) & 0xFFFF; - - if (Imm16 != ChunkVal) - break; - } - - // Create the second MOVK instruction. - MachineInstrBuilder MIB2 = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi)) - .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead)) - .addReg(DstReg) - .addImm(Imm16) - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt)); - - transferImpOps(MI, MIB, MIB2); - MI.eraseFromParent(); - return true; - } - - return false; -} - -/// Check whether this chunk matches the pattern '1...0...'. This pattern -/// starts a contiguous sequence of ones if we look at the bits from the LSB -/// towards the MSB. -static bool isStartChunk(uint64_t Chunk) { - if (Chunk == 0 || Chunk == std::numeric_limits<uint64_t>::max()) - return false; - - return isMask_64(~Chunk); -} - -/// Check whether this chunk matches the pattern '0...1...' This pattern -/// ends a contiguous sequence of ones if we look at the bits from the LSB -/// towards the MSB. -static bool isEndChunk(uint64_t Chunk) { - if (Chunk == 0 || Chunk == std::numeric_limits<uint64_t>::max()) - return false; - - return isMask_64(Chunk); -} - -/// Clear or set all bits in the chunk at the given index. -static uint64_t updateImm(uint64_t Imm, unsigned Idx, bool Clear) { - const uint64_t Mask = 0xFFFF; - - if (Clear) - // Clear chunk in the immediate. - Imm &= ~(Mask << (Idx * 16)); - else - // Set all bits in the immediate for the particular chunk. - Imm |= Mask << (Idx * 16); - - return Imm; -} - -/// Check whether the constant contains a sequence of contiguous ones, -/// which might be interrupted by one or two chunks. If so, materialize the -/// sequence of contiguous ones with an ORR instruction. -/// Materialize the chunks which are either interrupting the sequence or outside -/// of the sequence with a MOVK instruction. -/// -/// Assuming S is a chunk which starts the sequence (1...0...), E is a chunk -/// which ends the sequence (0...1...). Then we are looking for constants which -/// contain at least one S and E chunk. -/// E.g. |E|A|B|S|, |A|E|B|S| or |A|B|E|S|. -/// -/// We are also looking for constants like |S|A|B|E| where the contiguous -/// sequence of ones wraps around the MSB into the LSB. -static bool trySequenceOfOnes(uint64_t UImm, MachineInstr &MI, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator &MBBI, - const AArch64InstrInfo *TII) { - const int NotSet = -1; - const uint64_t Mask = 0xFFFF; - - int StartIdx = NotSet; - int EndIdx = NotSet; - // Try to find the chunks which start/end a contiguous sequence of ones. - for (int Idx = 0; Idx < 4; ++Idx) { - int64_t Chunk = getChunk(UImm, Idx); - // Sign extend the 16-bit chunk to 64-bit. - Chunk = (Chunk << 48) >> 48; - - if (isStartChunk(Chunk)) - StartIdx = Idx; - else if (isEndChunk(Chunk)) - EndIdx = Idx; - } - - // Early exit in case we can't find a start/end chunk. - if (StartIdx == NotSet || EndIdx == NotSet) - return false; - - // Outside of the contiguous sequence of ones everything needs to be zero. - uint64_t Outside = 0; - // Chunks between the start and end chunk need to have all their bits set. - uint64_t Inside = Mask; - - // If our contiguous sequence of ones wraps around from the MSB into the LSB, - // just swap indices and pretend we are materializing a contiguous sequence - // of zeros surrounded by a contiguous sequence of ones. - if (StartIdx > EndIdx) { - std::swap(StartIdx, EndIdx); - std::swap(Outside, Inside); - } - - uint64_t OrrImm = UImm; - int FirstMovkIdx = NotSet; - int SecondMovkIdx = NotSet; - - // Find out which chunks we need to patch up to obtain a contiguous sequence - // of ones. - for (int Idx = 0; Idx < 4; ++Idx) { - const uint64_t Chunk = getChunk(UImm, Idx); - - // Check whether we are looking at a chunk which is not part of the - // contiguous sequence of ones. - if ((Idx < StartIdx || EndIdx < Idx) && Chunk != Outside) { - OrrImm = updateImm(OrrImm, Idx, Outside == 0); - - // Remember the index we need to patch. - if (FirstMovkIdx == NotSet) - FirstMovkIdx = Idx; - else - SecondMovkIdx = Idx; - - // Check whether we are looking a chunk which is part of the contiguous - // sequence of ones. - } else if (Idx > StartIdx && Idx < EndIdx && Chunk != Inside) { - OrrImm = updateImm(OrrImm, Idx, Inside != Mask); - - // Remember the index we need to patch. - if (FirstMovkIdx == NotSet) - FirstMovkIdx = Idx; - else - SecondMovkIdx = Idx; - } - } - assert(FirstMovkIdx != NotSet && "Constant materializable with single ORR!"); - - // Create the ORR-immediate instruction. - uint64_t Encoding = 0; - AArch64_AM::processLogicalImmediate(OrrImm, 64, Encoding); - MachineInstrBuilder MIB = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXri)) - .add(MI.getOperand(0)) - .addReg(AArch64::XZR) - .addImm(Encoding); - - const unsigned DstReg = MI.getOperand(0).getReg(); - const bool DstIsDead = MI.getOperand(0).isDead(); - - const bool SingleMovk = SecondMovkIdx == NotSet; - // Create the first MOVK instruction. - MachineInstrBuilder MIB1 = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi)) - .addReg(DstReg, - RegState::Define | getDeadRegState(DstIsDead && SingleMovk)) - .addReg(DstReg) - .addImm(getChunk(UImm, FirstMovkIdx)) - .addImm( - AArch64_AM::getShifterImm(AArch64_AM::LSL, FirstMovkIdx * 16)); - - // Early exit in case we only need to emit a single MOVK instruction. - if (SingleMovk) { - transferImpOps(MI, MIB, MIB1); - MI.eraseFromParent(); - return true; - } - - // Create the second MOVK instruction. - MachineInstrBuilder MIB2 = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi)) - .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead)) - .addReg(DstReg) - .addImm(getChunk(UImm, SecondMovkIdx)) - .addImm( - AArch64_AM::getShifterImm(AArch64_AM::LSL, SecondMovkIdx * 16)); - - transferImpOps(MI, MIB, MIB2); - MI.eraseFromParent(); - return true; -} - /// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more /// real move-immediate instructions to synthesize the immediate. bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB, @@ -385,7 +111,6 @@ bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB, MachineInstr &MI = *MBBI; unsigned DstReg = MI.getOperand(0).getReg(); uint64_t Imm = MI.getOperand(1).getImm(); - const unsigned Mask = 0xFFFF; if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) { // Useless def, and we don't want to risk creating an invalid ORR (which @@ -394,194 +119,50 @@ bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB, return true; } - // Scan the immediate and count the number of 16-bit chunks which are either - // all ones or all zeros. - unsigned OneChunks = 0; - unsigned ZeroChunks = 0; - for (unsigned Shift = 0; Shift < BitSize; Shift += 16) { - const unsigned Chunk = (Imm >> Shift) & Mask; - if (Chunk == Mask) - OneChunks++; - else if (Chunk == 0) - ZeroChunks++; - } - - // FIXME: Prefer MOVZ/MOVN over ORR because of the rules for the "mov" - // alias. - - // Try a single ORR. - uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize); - uint64_t Encoding; - if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) { - unsigned Opc = (BitSize == 32 ? AArch64::ORRWri : AArch64::ORRXri); - MachineInstrBuilder MIB = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)) - .add(MI.getOperand(0)) - .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR) - .addImm(Encoding); - transferImpOps(MI, MIB, MIB); - MI.eraseFromParent(); - return true; - } - - // Two instruction sequences. - // - // Prefer MOVZ/MOVN followed by MOVK; it's more readable, and possibly the - // fastest sequence with fast literal generation. - if (OneChunks >= (BitSize / 16) - 2 || ZeroChunks >= (BitSize / 16) - 2) - return expandMOVImmSimple(MBB, MBBI, BitSize, OneChunks, ZeroChunks); + SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn; + AArch64_IMM::expandMOVImm(Imm, BitSize, Insn); + assert(Insn.size() != 0); - assert(BitSize == 64 && "All 32-bit immediates can be expanded with a" - "MOVZ/MOVK pair"); + SmallVector<MachineInstrBuilder, 4> MIBS; + for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) { + bool LastItem = std::next(I) == E; + switch (I->Opcode) + { + default: llvm_unreachable("unhandled!"); break; - // Try other two-instruction sequences. - - // 64-bit ORR followed by MOVK. - // We try to construct the ORR immediate in three different ways: either we - // zero out the chunk which will be replaced, we fill the chunk which will - // be replaced with ones, or we take the bit pattern from the other half of - // the 64-bit immediate. This is comprehensive because of the way ORR - // immediates are constructed. - for (unsigned Shift = 0; Shift < BitSize; Shift += 16) { - uint64_t ShiftedMask = (0xFFFFULL << Shift); - uint64_t ZeroChunk = UImm & ~ShiftedMask; - uint64_t OneChunk = UImm | ShiftedMask; - uint64_t RotatedImm = (UImm << 32) | (UImm >> 32); - uint64_t ReplicateChunk = ZeroChunk | (RotatedImm & ShiftedMask); - if (AArch64_AM::processLogicalImmediate(ZeroChunk, BitSize, Encoding) || - AArch64_AM::processLogicalImmediate(OneChunk, BitSize, Encoding) || - AArch64_AM::processLogicalImmediate(ReplicateChunk, - BitSize, Encoding)) { - // Create the ORR-immediate instruction. - MachineInstrBuilder MIB = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXri)) - .add(MI.getOperand(0)) - .addReg(AArch64::XZR) - .addImm(Encoding); - - // Create the MOVK instruction. - const unsigned Imm16 = getChunk(UImm, Shift / 16); - const unsigned DstReg = MI.getOperand(0).getReg(); - const bool DstIsDead = MI.getOperand(0).isDead(); - MachineInstrBuilder MIB1 = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi)) - .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead)) - .addReg(DstReg) - .addImm(Imm16) - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift)); - - transferImpOps(MI, MIB, MIB1); - MI.eraseFromParent(); - return true; + case AArch64::ORRWri: + case AArch64::ORRXri: + MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode)) + .add(MI.getOperand(0)) + .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR) + .addImm(I->Op2)); + break; + case AArch64::MOVNWi: + case AArch64::MOVNXi: + case AArch64::MOVZWi: + case AArch64::MOVZXi: { + bool DstIsDead = MI.getOperand(0).isDead(); + MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode)) + .addReg(DstReg, RegState::Define | + getDeadRegState(DstIsDead && LastItem)) + .addImm(I->Op1) + .addImm(I->Op2)); + } break; + case AArch64::MOVKWi: + case AArch64::MOVKXi: { + unsigned DstReg = MI.getOperand(0).getReg(); + bool DstIsDead = MI.getOperand(0).isDead(); + MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode)) + .addReg(DstReg, + RegState::Define | + getDeadRegState(DstIsDead && LastItem)) + .addReg(DstReg) + .addImm(I->Op1) + .addImm(I->Op2)); + } break; } } - - // FIXME: Add more two-instruction sequences. - - // Three instruction sequences. - // - // Prefer MOVZ/MOVN followed by two MOVK; it's more readable, and possibly - // the fastest sequence with fast literal generation. (If neither MOVK is - // part of a fast literal generation pair, it could be slower than the - // four-instruction sequence, but we won't worry about that for now.) - if (OneChunks || ZeroChunks) - return expandMOVImmSimple(MBB, MBBI, BitSize, OneChunks, ZeroChunks); - - // Check for identical 16-bit chunks within the constant and if so materialize - // them with a single ORR instruction. The remaining one or two 16-bit chunks - // will be materialized with MOVK instructions. - if (BitSize == 64 && tryToreplicateChunks(UImm, MI, MBB, MBBI, TII)) - return true; - - // Check whether the constant contains a sequence of contiguous ones, which - // might be interrupted by one or two chunks. If so, materialize the sequence - // of contiguous ones with an ORR instruction. Materialize the chunks which - // are either interrupting the sequence or outside of the sequence with a - // MOVK instruction. - if (BitSize == 64 && trySequenceOfOnes(UImm, MI, MBB, MBBI, TII)) - return true; - - // We found no possible two or three instruction sequence; use the general - // four-instruction sequence. - return expandMOVImmSimple(MBB, MBBI, BitSize, OneChunks, ZeroChunks); -} - -/// \brief Expand a MOVi32imm or MOVi64imm pseudo instruction to a -/// MOVZ or MOVN of width BitSize followed by up to 3 MOVK instructions. -bool AArch64ExpandPseudo::expandMOVImmSimple(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - unsigned BitSize, - unsigned OneChunks, - unsigned ZeroChunks) { - MachineInstr &MI = *MBBI; - unsigned DstReg = MI.getOperand(0).getReg(); - uint64_t Imm = MI.getOperand(1).getImm(); - const unsigned Mask = 0xFFFF; - - // Use a MOVZ or MOVN instruction to set the high bits, followed by one or - // more MOVK instructions to insert additional 16-bit portions into the - // lower bits. - bool isNeg = false; - - // Use MOVN to materialize the high bits if we have more all one chunks - // than all zero chunks. - if (OneChunks > ZeroChunks) { - isNeg = true; - Imm = ~Imm; - } - - unsigned FirstOpc; - if (BitSize == 32) { - Imm &= (1LL << 32) - 1; - FirstOpc = (isNeg ? AArch64::MOVNWi : AArch64::MOVZWi); - } else { - FirstOpc = (isNeg ? AArch64::MOVNXi : AArch64::MOVZXi); - } - unsigned Shift = 0; // LSL amount for high bits with MOVZ/MOVN - unsigned LastShift = 0; // LSL amount for last MOVK - if (Imm != 0) { - unsigned LZ = countLeadingZeros(Imm); - unsigned TZ = countTrailingZeros(Imm); - Shift = (TZ / 16) * 16; - LastShift = ((63 - LZ) / 16) * 16; - } - unsigned Imm16 = (Imm >> Shift) & Mask; - bool DstIsDead = MI.getOperand(0).isDead(); - MachineInstrBuilder MIB1 = - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(FirstOpc)) - .addReg(DstReg, RegState::Define | - getDeadRegState(DstIsDead && Shift == LastShift)) - .addImm(Imm16) - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift)); - - // If a MOVN was used for the high bits of a negative value, flip the rest - // of the bits back for use with MOVK. - if (isNeg) - Imm = ~Imm; - - if (Shift == LastShift) { - transferImpOps(MI, MIB1, MIB1); - MI.eraseFromParent(); - return true; - } - - MachineInstrBuilder MIB2; - unsigned Opc = (BitSize == 32 ? AArch64::MOVKWi : AArch64::MOVKXi); - while (Shift < LastShift) { - Shift += 16; - Imm16 = (Imm >> Shift) & Mask; - if (Imm16 == (isNeg ? Mask : 0)) - continue; // This 16-bit portion is already set correctly. - MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)) - .addReg(DstReg, - RegState::Define | - getDeadRegState(DstIsDead && Shift == LastShift)) - .addReg(DstReg) - .addImm(Imm16) - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift)); - } - - transferImpOps(MI, MIB1, MIB2); + transferImpOps(MI, MIBS.front(), MIBS.back()); MI.eraseFromParent(); return true; } @@ -759,6 +340,64 @@ bool AArch64ExpandPseudo::expandCMP_SWAP_128( return true; } +bool AArch64ExpandPseudo::expandSetTagLoop( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + Register SizeReg = MI.getOperand(2).getReg(); + Register AddressReg = MI.getOperand(3).getReg(); + + MachineFunction *MF = MBB.getParent(); + + bool ZeroData = MI.getOpcode() == AArch64::STZGloop; + const unsigned OpCode = + ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex; + + auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); + auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); + + MF->insert(++MBB.getIterator(), LoopBB); + MF->insert(++LoopBB->getIterator(), DoneBB); + + BuildMI(LoopBB, DL, TII->get(OpCode)) + .addDef(AddressReg) + .addReg(AddressReg) + .addReg(AddressReg) + .addImm(2) + .cloneMemRefs(MI) + .setMIFlags(MI.getFlags()); + BuildMI(LoopBB, DL, TII->get(AArch64::SUBXri)) + .addDef(SizeReg) + .addReg(SizeReg) + .addImm(16 * 2) + .addImm(0); + BuildMI(LoopBB, DL, TII->get(AArch64::CBNZX)).addUse(SizeReg).addMBB(LoopBB); + + LoopBB->addSuccessor(LoopBB); + LoopBB->addSuccessor(DoneBB); + + DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end()); + DoneBB->transferSuccessors(&MBB); + + MBB.addSuccessor(LoopBB); + + NextMBBI = MBB.end(); + MI.eraseFromParent(); + // Recompute liveness bottom up. + LivePhysRegs LiveRegs; + computeAndAddLiveIns(LiveRegs, *DoneBB); + computeAndAddLiveIns(LiveRegs, *LoopBB); + // Do an extra pass in the loop to get the loop carried dependencies right. + // FIXME: is this necessary? + LoopBB->clearLiveIns(); + computeAndAddLiveIns(LiveRegs, *LoopBB); + DoneBB->clearLiveIns(); + computeAndAddLiveIns(LiveRegs, *DoneBB); + + return true; +} + /// If MBBI references a pseudo instruction that should be expanded here, /// do the expansion and return true. Otherwise return false. bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB, @@ -928,6 +567,12 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB, if (MF->getTarget().getTargetTriple().isOSFuchsia() && MF->getTarget().getCodeModel() == CodeModel::Kernel) SysReg = AArch64SysReg::TPIDR_EL1; + else if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP()) + SysReg = AArch64SysReg::TPIDR_EL3; + else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP()) + SysReg = AArch64SysReg::TPIDR_EL2; + else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP()) + SysReg = AArch64SysReg::TPIDR_EL1; BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg) .addImm(SysReg); MI.eraseFromParent(); @@ -986,6 +631,46 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB, MI.eraseFromParent(); return true; } + case AArch64::IRGstack: { + MachineFunction &MF = *MBB.getParent(); + const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); + const AArch64FrameLowering *TFI = + MF.getSubtarget<AArch64Subtarget>().getFrameLowering(); + + // IRG does not allow immediate offset. getTaggedBasePointerOffset should + // almost always point to SP-after-prologue; if not, emit a longer + // instruction sequence. + int BaseOffset = -AFI->getTaggedBasePointerOffset(); + unsigned FrameReg; + int FrameRegOffset = TFI->resolveFrameOffsetReference( + MF, BaseOffset, false /*isFixed*/, FrameReg, /*PreferFP=*/false, + /*ForSimm=*/true); + Register SrcReg = FrameReg; + if (FrameRegOffset != 0) { + // Use output register as temporary. + SrcReg = MI.getOperand(0).getReg(); + emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg, + FrameRegOffset, TII); + } + BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG)) + .add(MI.getOperand(0)) + .addUse(SrcReg) + .add(MI.getOperand(2)); + MI.eraseFromParent(); + return true; + } + case AArch64::TAGPstack: { + BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDG)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(4)); + MI.eraseFromParent(); + return true; + } + case AArch64::STGloop: + case AArch64::STZGloop: + return expandSetTagLoop(MBB, MBBI, NextMBBI); } return false; } diff --git a/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp b/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp index bc9a5ca97fea..3b3182128c4c 100644 --- a/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp +++ b/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp @@ -1,9 +1,8 @@ //===- AArch64FalkorHWPFFix.cpp - Avoid HW prefetcher pitfalls on Falkor --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file For Falkor, we want to avoid HW prefetcher instruction tag collisions @@ -213,8 +212,8 @@ private: struct LoadInfo { LoadInfo() = default; - unsigned DestReg = 0; - unsigned BaseReg = 0; + Register DestReg; + Register BaseReg; int BaseRegIdx = -1; const MachineOperand *OffsetOpnd = nullptr; bool IsPrePost = false; @@ -648,7 +647,7 @@ static Optional<LoadInfo> getLoadInfo(const MachineInstr &MI) { return None; LoadInfo LI; - LI.DestReg = DestRegIdx == -1 ? 0 : MI.getOperand(DestRegIdx).getReg(); + LI.DestReg = DestRegIdx == -1 ? Register() : MI.getOperand(DestRegIdx).getReg(); LI.BaseReg = BaseReg; LI.BaseRegIdx = BaseRegIdx; LI.OffsetOpnd = OffsetIdx == -1 ? nullptr : &MI.getOperand(OffsetIdx); diff --git a/lib/Target/AArch64/AArch64FastISel.cpp b/lib/Target/AArch64/AArch64FastISel.cpp index 47550cabb9f0..8dc2768b9597 100644 --- a/lib/Target/AArch64/AArch64FastISel.cpp +++ b/lib/Target/AArch64/AArch64FastISel.cpp @@ -1,9 +1,8 @@ //===- AArch6464FastISel.cpp - AArch64 FastISel implementation ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -305,8 +304,6 @@ public: } // end anonymous namespace -#include "AArch64GenCallingConv.inc" - /// Check if the sign-/zero-extend will be a noop. static bool isIntExtFree(const Instruction *I) { assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) && @@ -408,10 +405,9 @@ unsigned AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) { bool Is64Bit = (VT == MVT::f64); // This checks to see if we can use FMOV instructions to materialize // a constant, otherwise we have to materialize via the constant pool. - if (TLI.isFPImmLegal(Val, VT)) { - int Imm = - Is64Bit ? AArch64_AM::getFP64Imm(Val) : AArch64_AM::getFP32Imm(Val); - assert((Imm != -1) && "Cannot encode floating-point constant."); + int Imm = + Is64Bit ? AArch64_AM::getFP64Imm(Val) : AArch64_AM::getFP32Imm(Val); + if (Imm != -1) { unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi; return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm); } @@ -2369,7 +2365,7 @@ bool AArch64FastISel::emitCompareAndBranch(const BranchInst *BI) { AArch64::sub_32); if ((BW < 32) && !IsBitTest) - SrcReg = emitIntExt(VT, SrcReg, MVT::i32, /*IsZExt=*/true); + SrcReg = emitIntExt(VT, SrcReg, MVT::i32, /*isZExt=*/true); // Emit the combined compare and branch instruction. SrcReg = constrainOperandRegClass(II, SrcReg, II.getNumDefs()); @@ -3608,6 +3604,14 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK)) .addImm(1); return true; + case Intrinsic::debugtrap: { + if (Subtarget->isTargetWindows()) { + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK)) + .addImm(0xF000); + return true; + } + break; + } case Intrinsic::sqrt: { Type *RetTy = II->getCalledFunction()->getReturnType(); @@ -4268,7 +4272,7 @@ unsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg, bool Op0IsKill, const TargetRegisterClass *RC = (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; if (NeedTrunc) { - Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32, /*IsZExt=*/false); + Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32, /*isZExt=*/false); Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask); Op0IsKill = Op1IsKill = true; } @@ -4948,7 +4952,7 @@ std::pair<unsigned, bool> AArch64FastISel::getRegForGEPIndex(const Value *Idx) { MVT PtrVT = TLI.getPointerTy(DL); EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false); if (IdxVT.bitsLT(PtrVT)) { - IdxN = emitIntExt(IdxVT.getSimpleVT(), IdxN, PtrVT, /*IsZExt=*/false); + IdxN = emitIntExt(IdxVT.getSimpleVT(), IdxN, PtrVT, /*isZExt=*/false); IdxNIsKill = true; } else if (IdxVT.bitsGT(PtrVT)) llvm_unreachable("AArch64 FastISel doesn't support types larger than i64"); @@ -5172,10 +5176,6 @@ bool AArch64FastISel::fastSelectInstruction(const Instruction *I) { return selectAtomicCmpXchg(cast<AtomicCmpXchgInst>(I)); } - // Silence warnings. - (void)&CC_AArch64_DarwinPCS_VarArg; - (void)&CC_AArch64_Win64_VarArg; - // fall-back to target-independent instruction selection. return selectOperator(I, I->getOpcode()); } diff --git a/lib/Target/AArch64/AArch64FrameLowering.cpp b/lib/Target/AArch64/AArch64FrameLowering.cpp index 538a8d7e8fbc..8c6e5cbd5c13 100644 --- a/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -1,9 +1,8 @@ //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -251,8 +250,7 @@ MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr( bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode(); uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0; - const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); - if (!TFI->hasReservedCallFrame(MF)) { + if (!hasReservedCallFrame(MF)) { unsigned Align = getStackAlignment(); int64_t Amount = I->getOperand(0).getImm(); @@ -588,7 +586,7 @@ static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, - bool NeedsWinCFI, bool InProlog = true) { + bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) { // Ignore instructions that do not operate on SP, i.e. shadow call stack // instructions and associated CFI instruction. while (MBBI->getOpcode() == AArch64::STRXpost || @@ -674,9 +672,11 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( MIB.setMemRefs(MBBI->memoperands()); // Generate a new SEH code that corresponds to the new instruction. - if (NeedsWinCFI) + if (NeedsWinCFI) { + *HasWinCFI = true; InsertSEH(*MIB, *TII, InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy); + } return std::prev(MBB.erase(MBBI)); } @@ -685,7 +685,8 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( // combined SP bump by adding the local stack size to the stack offsets. static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, unsigned LocalStackSize, - bool NeedsWinCFI) { + bool NeedsWinCFI, + bool *HasWinCFI) { if (AArch64InstrInfo::isSEHInstruction(MI)) return; @@ -732,6 +733,7 @@ static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale); if (NeedsWinCFI) { + *HasWinCFI = true; auto MBBI = std::next(MachineBasicBlock::iterator(MI)); assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction"); assert(AArch64InstrInfo::isSEHInstruction(*MBBI) && @@ -803,7 +805,9 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, !MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); bool HasFP = hasFP(MF); bool NeedsWinCFI = needsWinCFI(MF); - MF.setHasWinCFI(NeedsWinCFI); + bool HasWinCFI = false; + auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); }); + bool IsFunclet = MBB.isEHFuncletEntry(); // At this point, we're going to decide whether or not the function uses a @@ -838,6 +842,10 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, if (MF.getFunction().getCallingConv() == CallingConv::GHC) return; + // Set tagged base pointer to the bottom of the stack frame. + // Ideally it should match SP value after prologue. + AFI->setTaggedBasePointerOffset(MFI.getStackSize()); + // getStackSize() includes all the locals in its size calculation. We don't // include these locals when computing the stack size of a funclet, as they // are allocated in the parent's stack frame and accessed via the frame @@ -859,7 +867,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, ++NumRedZoneFunctions; } else { emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII, - MachineInstr::FrameSetup, false, NeedsWinCFI); + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); if (!NeedsWinCFI) { // Label used to tie together the PROLOG_LABEL and the MachineMoves. MCSymbol *FrameLabel = MMI.getContext().createTempSymbol(); @@ -872,9 +880,11 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, } } - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) .setMIFlag(MachineInstr::FrameSetup); + } return; } @@ -892,11 +902,11 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes); if (CombineSPBump) { emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII, - MachineInstr::FrameSetup, false, NeedsWinCFI); + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); NumBytes = 0; } else if (PrologueSaveSize != 0) { MBBI = convertCalleeSaveRestoreToSPPrePostIncDec( - MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI); + MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI); NumBytes -= PrologueSaveSize; } assert(NumBytes >= 0 && "Negative stack allocation size!?"); @@ -908,7 +918,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) { if (CombineSPBump) fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(), - NeedsWinCFI); + NeedsWinCFI, &HasWinCFI); ++MBBI; } @@ -916,9 +926,24 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, // opcodes that we needed to emit. The FP and BP belong to the containing // function. if (IsFunclet) { - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) .setMIFlag(MachineInstr::FrameSetup); + } + + // SEH funclets are passed the frame pointer in X1. If the parent + // function uses the base register, then the base register is used + // directly, and is not retrieved from X1. + if (F.hasPersonalityFn()) { + EHPersonality Per = classifyEHPersonality(F.getPersonalityFn()); + if (isAsynchronousEHPersonality(Per)) { + BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP) + .addReg(AArch64::X1).setMIFlag(MachineInstr::FrameSetup); + MBB.addLiveIn(AArch64::X1); + } + } + return; } @@ -934,12 +959,13 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, // Note: All stores of callee-saved registers are marked as "FrameSetup". // This code marks the instruction(s) that set the FP also. emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII, - MachineInstr::FrameSetup, false, NeedsWinCFI); + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); } if (windowsRequiresStackProbe(MF, NumBytes)) { uint32_t NumWords = NumBytes >> 4; if (NeedsWinCFI) { + HasWinCFI = true; // alloc_l can hold at most 256MB, so assume that NumBytes doesn't // exceed this amount. We need to move at most 2^24 - 1 into x15. // This is at most two instructions, MOVZ follwed by MOVK. @@ -983,9 +1009,11 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); + } break; case CodeModel::Large: BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT)) @@ -993,9 +1021,11 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, .addExternalSymbol("__chkstk") .addExternalSymbol("__chkstk") .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); + } BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR)) .addReg(AArch64::X16, RegState::Kill) @@ -1004,9 +1034,11 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); + } break; } @@ -1015,10 +1047,12 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, .addReg(AArch64::X15, RegState::Kill) .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4)) .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) - .addImm(NumBytes) - .setMIFlag(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) + .addImm(NumBytes) + .setMIFlag(MachineInstr::FrameSetup); + } NumBytes = 0; } @@ -1038,7 +1072,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, // the correct value here, as NumBytes also includes padding bytes, // which shouldn't be counted here. emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII, - MachineInstr::FrameSetup, false, NeedsWinCFI); + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); if (NeedsRealignment) { const unsigned Alignment = MFI.getMaxAlignment(); @@ -1061,10 +1095,12 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, .addReg(scratchSPReg, RegState::Kill) .addImm(andMaskEncoded); AFI->setStackRealigned(true); - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) .addImm(NumBytes & andMaskEncoded) .setMIFlag(MachineInstr::FrameSetup); + } } } @@ -1078,16 +1114,19 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, if (RegInfo->hasBasePointer(MF)) { TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP, false); - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); + } } // The very last FrameSetup instruction indicates the end of prologue. Emit a // SEH opcode indicating the prologue end. - if (NeedsWinCFI) + if (NeedsWinCFI && HasWinCFI) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) .setMIFlag(MachineInstr::FrameSetup); + } if (needsFrameMoves) { const DataLayout &TD = MF.getDataLayout(); @@ -1231,7 +1270,12 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, DebugLoc DL; bool IsTailCallReturn = false; bool NeedsWinCFI = needsWinCFI(MF); + bool HasWinCFI = false; bool IsFunclet = false; + auto WinCFI = make_scope_exit([&]() { + if (!MF.hasWinCFI()) + MF.setHasWinCFI(HasWinCFI); + }); if (MBB.end() != MBBI) { DL = MBBI->getDebugLoc(); @@ -1326,7 +1370,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // If the offset is 0, convert it to a post-index ldp. if (OffsetOp.getImm() == 0) convertCalleeSaveRestoreToSPPrePostIncDec( - MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, false); + MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false); else { // If not, make sure to emit an add after the last ldp. // We're doing this by transfering the size to be restored from the @@ -1348,19 +1392,21 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, break; } else if (CombineSPBump) fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(), - NeedsWinCFI); + NeedsWinCFI, &HasWinCFI); } - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart)) .setMIFlag(MachineInstr::FrameDestroy); + } // If there is a single SP update, insert it before the ret and we're done. if (CombineSPBump) { emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP, NumBytes + AfterCSRPopSize, TII, MachineInstr::FrameDestroy, - false, NeedsWinCFI); - if (NeedsWinCFI) + false, NeedsWinCFI, &HasWinCFI); + if (NeedsWinCFI && HasWinCFI) BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd)) .setMIFlag(MachineInstr::FrameDestroy); @@ -1392,12 +1438,14 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, StackRestoreBytes, TII, MachineInstr::FrameDestroy, false, - NeedsWinCFI); + NeedsWinCFI, &HasWinCFI); if (Done) { - if (NeedsWinCFI) + if (NeedsWinCFI) { + HasWinCFI = true; BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd)) .setMIFlag(MachineInstr::FrameDestroy); + } return; } @@ -1436,11 +1484,13 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP, AfterCSRPopSize, TII, MachineInstr::FrameDestroy, false, - NeedsWinCFI); + NeedsWinCFI, &HasWinCFI); } - if (NeedsWinCFI) + if (NeedsWinCFI && HasWinCFI) BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd)) .setMIFlag(MachineInstr::FrameDestroy); + + MF.setHasWinCFI(HasWinCFI); } /// getFrameIndexReference - Provide a base+offset reference to an FI slot for @@ -1450,25 +1500,66 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const { - return resolveFrameIndexReference(MF, FI, FrameReg); + return resolveFrameIndexReference( + MF, FI, FrameReg, + /*PreferFP=*/ + MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress), + /*ForSimm=*/false); } -int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, - int FI, unsigned &FrameReg, - bool PreferFP) const { - const MachineFrameInfo &MFI = MF.getFrameInfo(); - const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>( - MF.getSubtarget().getRegisterInfo()); - const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); - const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); +int AArch64FrameLowering::getNonLocalFrameIndexReference( + const MachineFunction &MF, int FI) const { + return getSEHFrameIndexOffset(MF, FI); +} + +static int getFPOffset(const MachineFunction &MF, int ObjectOffset) { + const auto *AFI = MF.getInfo<AArch64FunctionInfo>(); + const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>(); bool IsWin64 = Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()); unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0; - int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16; - int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize(); + return ObjectOffset + FixedObject + 16; +} + +static int getStackOffset(const MachineFunction &MF, int ObjectOffset) { + const auto &MFI = MF.getFrameInfo(); + return ObjectOffset + MFI.getStackSize(); +} + +int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF, + int FI) const { + const auto *RegInfo = static_cast<const AArch64RegisterInfo *>( + MF.getSubtarget().getRegisterInfo()); + int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI); + return RegInfo->getLocalAddressRegister(MF) == AArch64::FP + ? getFPOffset(MF, ObjectOffset) + : getStackOffset(MF, ObjectOffset); +} + +int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, + int FI, unsigned &FrameReg, + bool PreferFP, + bool ForSimm) const { + const auto &MFI = MF.getFrameInfo(); + int ObjectOffset = MFI.getObjectOffset(FI); bool isFixed = MFI.isFixedObjectIndex(FI); - bool isCSR = !isFixed && MFI.getObjectOffset(FI) >= - -((int)AFI->getCalleeSavedStackSize()); + return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, FrameReg, + PreferFP, ForSimm); +} + +int AArch64FrameLowering::resolveFrameOffsetReference( + const MachineFunction &MF, int ObjectOffset, bool isFixed, + unsigned &FrameReg, bool PreferFP, bool ForSimm) const { + const auto &MFI = MF.getFrameInfo(); + const auto *RegInfo = static_cast<const AArch64RegisterInfo *>( + MF.getSubtarget().getRegisterInfo()); + const auto *AFI = MF.getInfo<AArch64FunctionInfo>(); + const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>(); + + int FPOffset = getFPOffset(MF, ObjectOffset); + int Offset = getStackOffset(MF, ObjectOffset); + bool isCSR = + !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize()); // Use frame pointer to reference fixed objects. Use it for locals if // there are VLAs or a dynamically realigned SP (and thus the SP isn't @@ -1489,11 +1580,11 @@ int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, assert(hasFP(MF) && "Re-aligned stack must have frame pointer"); UseFP = true; } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) { - // If the FPOffset is negative, we have to keep in mind that the - // available offset range for negative offsets is smaller than for - // positive ones. If an offset is - // available via the FP and the SP, use whichever is closest. - bool FPOffsetFits = FPOffset >= -256; + // If the FPOffset is negative and we're producing a signed immediate, we + // have to keep in mind that the available offset range for negative + // offsets is smaller than for positive ones. If an offset is available + // via the FP and the SP, use whichever is closest. + bool FPOffsetFits = !ForSimm || FPOffset >= -256; PreferFP |= Offset > -FPOffset; if (MFI.hasVarSizedObjects()) { @@ -1517,6 +1608,7 @@ int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, // Funclets access the locals contained in the parent's stack frame // via the frame pointer, so we have to use the FP in the parent // function. + (void) Subtarget; assert( Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) && "Funclets should only be present on Win64"); @@ -1759,8 +1851,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters( static_cast<char>(unsigned(dwarf::DW_OP_breg18)), static_cast<char>(-8) & 0x7f, // addend (sleb128) }; - unsigned CFIIndex = - MF.addFrameInst(MCCFIInstruction::createEscape(nullptr, CFIInst)); + unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape( + nullptr, StringRef(CFIInst, sizeof(CFIInst)))); BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) .setMIFlag(MachineInstr::FrameSetup); @@ -2104,9 +2196,6 @@ void AArch64FrameLowering::processFunctionBeforeFrameFinalized( while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) ++MBBI; - if (MBBI->isTerminator()) - return; - // Create an UnwindHelp object. int UnwindHelpFI = MFI.CreateStackObject(/*size*/8, /*alignment*/16, false); @@ -2114,8 +2203,10 @@ void AArch64FrameLowering::processFunctionBeforeFrameFinalized( // We need to store -2 into the UnwindHelp object at the start of the // function. DebugLoc DL; - RS->enterBasicBlock(MBB); - unsigned DstReg = RS->scavengeRegister(&AArch64::GPR64RegClass, MBBI, 0); + RS->enterBasicBlockEnd(MBB); + RS->backward(std::prev(MBBI)); + unsigned DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass); + assert(DstReg && "There must be a free register after frame setup"); BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2); BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi)) .addReg(DstReg, getKillRegState(true)) diff --git a/lib/Target/AArch64/AArch64FrameLowering.h b/lib/Target/AArch64/AArch64FrameLowering.h index 0d0385acf46e..6dbd34b2189f 100644 --- a/lib/Target/AArch64/AArch64FrameLowering.h +++ b/lib/Target/AArch64/AArch64FrameLowering.h @@ -1,9 +1,8 @@ //==-- AArch64FrameLowering.h - TargetFrameLowering for AArch64 --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -41,8 +40,11 @@ public: int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const override; int resolveFrameIndexReference(const MachineFunction &MF, int FI, - unsigned &FrameReg, - bool PreferFP = false) const; + unsigned &FrameReg, bool PreferFP, + bool ForSimm) const; + int resolveFrameOffsetReference(const MachineFunction &MF, int ObjectOffset, + bool isFixed, unsigned &FrameReg, + bool PreferFP, bool ForSimm) const; bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, @@ -79,6 +81,9 @@ public: int getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, unsigned &FrameReg, bool IgnoreSPUpdates) const override; + int getNonLocalFrameIndexReference(const MachineFunction &MF, + int FI) const override; + int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const; private: bool shouldCombineCSRLocalStackBump(MachineFunction &MF, diff --git a/lib/Target/AArch64/AArch64GenRegisterBankInfo.def b/lib/Target/AArch64/AArch64GenRegisterBankInfo.def index 37720cbd32bb..528756b34856 100644 --- a/lib/Target/AArch64/AArch64GenRegisterBankInfo.def +++ b/lib/Target/AArch64/AArch64GenRegisterBankInfo.def @@ -1,9 +1,8 @@ //===- AArch64GenRegisterBankInfo.def ----------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -111,6 +110,10 @@ RegisterBankInfo::ValueMapping AArch64GenRegisterBankInfo::ValMappings[]{ // 47: FPExt vector: 64 to 128. <-- This must match FPExt64To128Idx. {&AArch64GenRegisterBankInfo::PartMappings[PMI_FPR128 - PMI_Min], 1}, {&AArch64GenRegisterBankInfo::PartMappings[PMI_FPR64 - PMI_Min], 1}, + // 49: Shift scalar with 64 bit shift imm + {&AArch64GenRegisterBankInfo::PartMappings[PMI_GPR32 - PMI_Min], 1}, + {&AArch64GenRegisterBankInfo::PartMappings[PMI_GPR32 - PMI_Min], 1}, + {&AArch64GenRegisterBankInfo::PartMappings[PMI_GPR64 - PMI_Min], 1}, }; bool AArch64GenRegisterBankInfo::checkPartialMap(unsigned Idx, diff --git a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index fc9855f6a0da..cd7e927ac80c 100644 --- a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -1,9 +1,8 @@ //===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -53,7 +52,7 @@ public: } bool runOnMachineFunction(MachineFunction &MF) override { - ForCodeSize = MF.getFunction().optForSize(); + ForCodeSize = MF.getFunction().hasOptSize(); Subtarget = &MF.getSubtarget<AArch64Subtarget>(); return SelectionDAGISel::runOnMachineFunction(MF); } @@ -92,6 +91,12 @@ public: bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) { return SelectAddrModeIndexed7S(N, 16, Base, OffImm); } + bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) { + return SelectAddrModeIndexedBitWidth(N, true, 9, 16, Base, OffImm); + } + bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) { + return SelectAddrModeIndexedBitWidth(N, false, 6, 16, Base, OffImm); + } bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) { return SelectAddrModeIndexed(N, 1, Base, OffImm); } @@ -152,6 +157,9 @@ public: bool tryIndexedLoad(SDNode *N); + bool trySelectStackSlotTagP(SDNode *N); + void SelectTagP(SDNode *N); + void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc, unsigned SubRegIdx); void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc, @@ -180,7 +188,12 @@ private: bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg, SDValue &Shift); bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base, - SDValue &OffImm); + SDValue &OffImm) { + return SelectAddrModeIndexedBitWidth(N, true, 7, Size, Base, OffImm); + } + bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW, + unsigned Size, SDValue &Base, + SDValue &OffImm); bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base, SDValue &OffImm); bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base, @@ -676,12 +689,13 @@ static bool isWorthFoldingADDlow(SDValue N) { return true; } -/// SelectAddrModeIndexed7S - Select a "register plus scaled signed 7-bit +/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit /// immediate" address. The "Size" argument is the size in bytes of the memory /// reference, which determines the scale. -bool AArch64DAGToDAGISel::SelectAddrModeIndexed7S(SDValue N, unsigned Size, - SDValue &Base, - SDValue &OffImm) { +bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, + unsigned BW, unsigned Size, + SDValue &Base, + SDValue &OffImm) { SDLoc dl(N); const DataLayout &DL = CurDAG->getDataLayout(); const TargetLowering *TLI = getTargetLowering(); @@ -692,26 +706,43 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexed7S(SDValue N, unsigned Size, return true; } - // As opposed to the (12-bit) Indexed addressing mode below, the 7-bit signed + // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed // selected here doesn't support labels/immediates, only base+offset. - if (CurDAG->isBaseWithConstantOffset(N)) { if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { - int64_t RHSC = RHS->getSExtValue(); - unsigned Scale = Log2_32(Size); - if ((RHSC & (Size - 1)) == 0 && RHSC >= -(0x40 << Scale) && - RHSC < (0x40 << Scale)) { - Base = N.getOperand(0); - if (Base.getOpcode() == ISD::FrameIndex) { - int FI = cast<FrameIndexSDNode>(Base)->getIndex(); - Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); + if (IsSignedImm) { + int64_t RHSC = RHS->getSExtValue(); + unsigned Scale = Log2_32(Size); + int64_t Range = 0x1LL << (BW - 1); + + if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) && + RHSC < (Range << Scale)) { + Base = N.getOperand(0); + if (Base.getOpcode() == ISD::FrameIndex) { + int FI = cast<FrameIndexSDNode>(Base)->getIndex(); + Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); + } + OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); + return true; + } + } else { + // unsigned Immediate + uint64_t RHSC = RHS->getZExtValue(); + unsigned Scale = Log2_32(Size); + uint64_t Range = 0x1ULL << BW; + + if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) { + Base = N.getOperand(0); + if (Base.getOpcode() == ISD::FrameIndex) { + int FI = cast<FrameIndexSDNode>(Base)->getIndex(); + Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); + } + OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); + return true; } - OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); - return true; } } } - // Base only. The address will be materialized into a register before // the memory is accessed. // add x0, Xbase, #offset @@ -2650,6 +2681,14 @@ bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) { return true; } + if (RegString->getString() == "pc") { + ReplaceNode(N, CurDAG->getMachineNode( + AArch64::ADR, DL, N->getSimpleValueType(0), MVT::Other, + CurDAG->getTargetConstant(0, DL, MVT::i32), + N->getOperand(0))); + return true; + } + return false; } @@ -2754,6 +2793,58 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) { return true; } +bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) { + // tagp(FrameIndex, IRGstack, tag_offset): + // since the offset between FrameIndex and IRGstack is a compile-time + // constant, this can be lowered to a single ADDG instruction. + if (!(isa<FrameIndexSDNode>(N->getOperand(1)))) { + return false; + } + + SDValue IRG_SP = N->getOperand(2); + if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN || + cast<ConstantSDNode>(IRG_SP->getOperand(1))->getZExtValue() != + Intrinsic::aarch64_irg_sp) { + return false; + } + + const TargetLowering *TLI = getTargetLowering(); + SDLoc DL(N); + int FI = cast<FrameIndexSDNode>(N->getOperand(1))->getIndex(); + SDValue FiOp = CurDAG->getTargetFrameIndex( + FI, TLI->getPointerTy(CurDAG->getDataLayout())); + int TagOffset = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(); + + SDNode *Out = CurDAG->getMachineNode( + AArch64::TAGPstack, DL, MVT::i64, + {FiOp, CurDAG->getTargetConstant(0, DL, MVT::i64), N->getOperand(2), + CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); + ReplaceNode(N, Out); + return true; +} + +void AArch64DAGToDAGISel::SelectTagP(SDNode *N) { + assert(isa<ConstantSDNode>(N->getOperand(3)) && + "llvm.aarch64.tagp third argument must be an immediate"); + if (trySelectStackSlotTagP(N)) + return; + // FIXME: above applies in any case when offset between Op1 and Op2 is a + // compile-time constant, not just for stack allocations. + + // General case for unrelated pointers in Op1 and Op2. + SDLoc DL(N); + int TagOffset = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(); + SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP, DL, MVT::i64, + {N->getOperand(1), N->getOperand(2)}); + SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr, DL, MVT::i64, + {SDValue(N1, 0), N->getOperand(2)}); + SDNode *N3 = CurDAG->getMachineNode( + AArch64::ADDG, DL, MVT::i64, + {SDValue(N2, 0), CurDAG->getTargetConstant(0, DL, MVT::i64), + CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); + ReplaceNode(N, N3); +} + void AArch64DAGToDAGISel::Select(SDNode *Node) { // If we have a custom node, we already have selected! if (Node->isMachineOpcode()) { @@ -3247,6 +3338,9 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) { switch (IntNo) { default: break; + case Intrinsic::aarch64_tagp: + SelectTagP(Node); + return; case Intrinsic::aarch64_neon_tbl2: SelectTable(Node, 2, VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two, diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp index e01ca14d7f63..7becc99fb5c7 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1,9 +1,8 @@ //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -11,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "AArch64ExpandImm.h" #include "AArch64ISelLowering.h" #include "AArch64CallingConvention.h" #include "AArch64MachineFunctionInfo.h" @@ -55,9 +55,11 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Module.h" #include "llvm/IR/OperandTraits.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" #include "llvm/IR/Value.h" @@ -87,6 +89,7 @@ #include <vector> using namespace llvm; +using namespace llvm::PatternMatch; #define DEBUG_TYPE "aarch64-lower" @@ -454,6 +457,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FMAXNUM, Ty, Legal); setOperationAction(ISD::FMINIMUM, Ty, Legal); setOperationAction(ISD::FMAXIMUM, Ty, Legal); + setOperationAction(ISD::LROUND, Ty, Legal); + setOperationAction(ISD::LLROUND, Ty, Legal); + setOperationAction(ISD::LRINT, Ty, Legal); + setOperationAction(ISD::LLRINT, Ty, Legal); } if (Subtarget->hasFullFP16()) { @@ -544,9 +551,13 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // Trap. setOperationAction(ISD::TRAP, MVT::Other, Legal); + if (Subtarget->isTargetWindows()) + setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); // We combine OR nodes for bitfield operations. setTargetDAGCombine(ISD::OR); + // Try to create BICs for vector ANDs. + setTargetDAGCombine(ISD::AND); // Vector add and sub nodes may conceal a high-half opportunity. // Also, try to fold ADD into CSINC/CSINV.. @@ -608,9 +619,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setPrefLoopAlignment(STI.getPrefLoopAlignment()); // Only change the limit for entries in a jump table if specified by - // the subtarget, but not at the command line. + // the sub target, but not at the command line. unsigned MaxJT = STI.getMaximumJumpTableSize(); - if (MaxJT && getMaximumJumpTableSize() == 0) + if (MaxJT && getMaximumJumpTableSize() == UINT_MAX) setMaximumJumpTableSize(MaxJT); setHasExtractBitsInsn(true); @@ -658,14 +669,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // elements smaller than i32, so promote the input to i32 first. setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32); setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32); - setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32); - setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32); - // i8 and i16 vector elements also need promotion to i32 for v8i8 or v8i16 - // -> v8f16 conversions. + // i8 vector elements also need promotion to i32 for v8i8 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32); setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32); - setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32); - setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32); // Similarly, there is no direct i32 -> f64 vector conversion instruction. setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom); @@ -676,18 +682,23 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); + if (Subtarget->hasFullFP16()) { + setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); + setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); + setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom); + setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); + } else { + // when AArch64 doesn't have fullfp16 support, promote the input + // to i32 first. + setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32); + setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32); + setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32); + setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32); + } + setOperationAction(ISD::CTLZ, MVT::v1i64, Expand); setOperationAction(ISD::CTLZ, MVT::v2i64, Expand); - setOperationAction(ISD::CTTZ, MVT::v2i8, Expand); - setOperationAction(ISD::CTTZ, MVT::v4i16, Expand); - setOperationAction(ISD::CTTZ, MVT::v2i32, Expand); - setOperationAction(ISD::CTTZ, MVT::v1i64, Expand); - setOperationAction(ISD::CTTZ, MVT::v16i8, Expand); - setOperationAction(ISD::CTTZ, MVT::v8i16, Expand); - setOperationAction(ISD::CTTZ, MVT::v4i32, Expand); - setOperationAction(ISD::CTTZ, MVT::v2i64, Expand); - // AArch64 doesn't have MUL.2d: setOperationAction(ISD::MUL, MVT::v2i64, Expand); // Custom handling for some quad-vector types to detect MULL. @@ -696,14 +707,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::MUL, MVT::v2i64, Custom); // Vector reductions - for (MVT VT : MVT::integer_valuetypes()) { + for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, + MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); } - for (MVT VT : MVT::fp_valuetypes()) { + for (MVT VT : { MVT::v4f16, MVT::v2f32, + MVT::v8f16, MVT::v4f32, MVT::v2f64 }) { setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); } @@ -726,6 +739,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::UMUL_LOHI, VT, Expand); setOperationAction(ISD::BSWAP, VT, Expand); + setOperationAction(ISD::CTTZ, VT, Expand); for (MVT InnerVT : MVT::vector_valuetypes()) { setTruncStoreAction(VT, InnerVT, Expand); @@ -745,6 +759,17 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FROUND, Ty, Legal); } + if (Subtarget->hasFullFP16()) { + for (MVT Ty : {MVT::v4f16, MVT::v8f16}) { + setOperationAction(ISD::FFLOOR, Ty, Legal); + setOperationAction(ISD::FNEARBYINT, Ty, Legal); + setOperationAction(ISD::FCEIL, Ty, Legal); + setOperationAction(ISD::FRINT, Ty, Legal); + setOperationAction(ISD::FTRUNC, Ty, Legal); + setOperationAction(ISD::FROUND, Ty, Legal); + } + } + setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom); } @@ -783,7 +808,6 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) { setOperationAction(ISD::SRA, VT, Custom); setOperationAction(ISD::SRL, VT, Custom); setOperationAction(ISD::SHL, VT, Custom); - setOperationAction(ISD::AND, VT, Custom); setOperationAction(ISD::OR, VT, Custom); setOperationAction(ISD::SETCC, VT, Custom); setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); @@ -1052,10 +1076,9 @@ MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL, return MVT::i64; } -bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(EVT VT, - unsigned AddrSpace, - unsigned Align, - bool *Fast) const { +bool AArch64TargetLowering::allowsMisalignedMemoryAccesses( + EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags, + bool *Fast) const { if (Subtarget->requiresStrictAlign()) return false; @@ -1211,6 +1234,10 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const { case AArch64ISD::FRECPS: return "AArch64ISD::FRECPS"; case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE"; case AArch64ISD::FRSQRTS: return "AArch64ISD::FRSQRTS"; + case AArch64ISD::STG: return "AArch64ISD::STG"; + case AArch64ISD::STZG: return "AArch64ISD::STZG"; + case AArch64ISD::ST2G: return "AArch64ISD::ST2G"; + case AArch64ISD::STZ2G: return "AArch64ISD::STZ2G"; } return nullptr; } @@ -2326,7 +2353,8 @@ SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op, SDLoc(Op)).first; } -static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) { +SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op, + SelectionDAG &DAG) const { // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp. // Any additional optimization in this function should be recorded // in the cost tables. @@ -2334,8 +2362,9 @@ static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) { EVT VT = Op.getValueType(); unsigned NumElts = InVT.getVectorNumElements(); - // f16 vectors are promoted to f32 before a conversion. - if (InVT.getVectorElementType() == MVT::f16) { + // f16 conversions are promoted to f32 when full fp16 is not supported. + if (InVT.getVectorElementType() == MVT::f16 && + !Subtarget->hasFullFP16()) { MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts); SDLoc dl(Op); return DAG.getNode( @@ -2743,6 +2772,28 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::aarch64_neon_umin: return DAG.getNode(ISD::UMIN, dl, Op.getValueType(), Op.getOperand(1), Op.getOperand(2)); + + case Intrinsic::localaddress: { + const auto &MF = DAG.getMachineFunction(); + const auto *RegInfo = Subtarget->getRegisterInfo(); + unsigned Reg = RegInfo->getLocalAddressRegister(MF); + return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, + Op.getSimpleValueType()); + } + + case Intrinsic::eh_recoverfp: { + // FIXME: This needs to be implemented to correctly handle highly aligned + // stack objects. For now we simply return the incoming FP. Refer D53541 + // for more details. + SDValue FnOp = Op.getOperand(1); + SDValue IncomingFPOp = Op.getOperand(2); + GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp); + auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr); + if (!Fn) + report_fatal_error( + "llvm.eh.recoverfp must take a function as the first argument"); + return IncomingFPOp; + } } } @@ -2797,7 +2848,8 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op, unsigned AS = StoreNode->getAddressSpace(); unsigned Align = StoreNode->getAlignment(); if (Align < MemVT.getStoreSize() && - !allowsMisalignedMemoryAccesses(MemVT, AS, Align, nullptr)) { + !allowsMisalignedMemoryAccesses( + MemVT, AS, Align, StoreNode->getMemOperand()->getFlags(), nullptr)) { return scalarizeVectorStore(StoreNode, DAG); } @@ -2900,8 +2952,6 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op, return LowerCTPOP(Op, DAG); case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); - case ISD::AND: - return LowerVectorAND(Op, DAG); case ISD::OR: return LowerVectorOR(Op, DAG); case ISD::XOR: @@ -2945,8 +2995,6 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op, // Calling Convention Implementation //===----------------------------------------------------------------------===// -#include "AArch64GenCallingConv.inc" - /// Selects the correct CCAssignFn for a given CallingConvention value. CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const { @@ -3167,6 +3215,32 @@ SDValue AArch64TargetLowering::LowerFormalArguments( FuncInfo->getForwardedMustTailRegParms(); CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_AArch64_AAPCS); + + // Conservatively forward X8, since it might be used for aggregate return. + if (!CCInfo.isAllocated(AArch64::X8)) { + unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass); + Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64)); + } + } + } + + // On Windows, InReg pointers must be returned, so record the pointer in a + // virtual register at the start of the function so it can be returned in the + // epilogue. + if (IsWin64) { + for (unsigned I = 0, E = Ins.size(); I != E; ++I) { + if (Ins[I].Flags.isInReg()) { + assert(!FuncInfo->getSRetReturnReg()); + + MVT PtrTy = getPointerTy(DAG.getDataLayout()); + unsigned Reg = + MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy)); + FuncInfo->setSRetReturnReg(Reg); + + SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]); + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain); + break; + } } } @@ -3365,10 +3439,20 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization( // X86) but less efficient and uglier in LowerCall. for (Function::const_arg_iterator i = CallerF.arg_begin(), e = CallerF.arg_end(); - i != e; ++i) + i != e; ++i) { if (i->hasByValAttr()) return false; + // On Windows, "inreg" attributes signify non-aggregate indirect returns. + // In this case, it is necessary to save/restore X0 in the callee. Tail + // call opt interferes with this. So we disable tail call opt when the + // caller has an argument with "inreg" attribute. + + // FIXME: Check whether the callee also has an "inreg" argument. + if (i->hasInRegAttr()) + return false; + } + if (getTargetMachine().Options.GuaranteedTailCallOpt) return canGuaranteeTCO(CalleeCC) && CCMatch; @@ -3886,6 +3970,9 @@ AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, const SmallVectorImpl<ISD::OutputArg> &Outs, const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, SelectionDAG &DAG) const { + auto &MF = DAG.getMachineFunction(); + auto *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); + CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS : RetCC_AArch64_AAPCS; @@ -3924,6 +4011,23 @@ AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, Flag = Chain.getValue(1); RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); } + + // Windows AArch64 ABIs require that for returning structs by value we copy + // the sret argument into X0 for the return. + // We saved the argument into a virtual register in the entry block, + // so now we copy the value out and into X0. + if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) { + SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg, + getPointerTy(MF.getDataLayout())); + + unsigned RetValReg = AArch64::X0; + Chain = DAG.getCopyToReg(Chain, DL, RetValReg, Val, Flag); + Flag = Chain.getValue(1); + + RetOps.push_back( + DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout()))); + } + const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); const MCPhysReg *I = TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); @@ -5197,50 +5301,20 @@ SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op, return DAG.getFrameIndex(FI, VT); } +#define GET_REGISTER_MATCHER +#include "AArch64GenAsmMatcher.inc" + // FIXME? Maybe this could be a TableGen attribute on some registers and // this table could be generated automatically from RegInfo. unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT, SelectionDAG &DAG) const { - unsigned Reg = StringSwitch<unsigned>(RegName) - .Case("sp", AArch64::SP) - .Case("x1", AArch64::X1) - .Case("w1", AArch64::W1) - .Case("x2", AArch64::X2) - .Case("w2", AArch64::W2) - .Case("x3", AArch64::X3) - .Case("w3", AArch64::W3) - .Case("x4", AArch64::X4) - .Case("w4", AArch64::W4) - .Case("x5", AArch64::X5) - .Case("w5", AArch64::W5) - .Case("x6", AArch64::X6) - .Case("w6", AArch64::W6) - .Case("x7", AArch64::X7) - .Case("w7", AArch64::W7) - .Case("x18", AArch64::X18) - .Case("w18", AArch64::W18) - .Case("x20", AArch64::X20) - .Case("w20", AArch64::W20) - .Default(0); - if (((Reg == AArch64::X1 || Reg == AArch64::W1) && - !Subtarget->isXRegisterReserved(1)) || - ((Reg == AArch64::X2 || Reg == AArch64::W2) && - !Subtarget->isXRegisterReserved(2)) || - ((Reg == AArch64::X3 || Reg == AArch64::W3) && - !Subtarget->isXRegisterReserved(3)) || - ((Reg == AArch64::X4 || Reg == AArch64::W4) && - !Subtarget->isXRegisterReserved(4)) || - ((Reg == AArch64::X5 || Reg == AArch64::W5) && - !Subtarget->isXRegisterReserved(5)) || - ((Reg == AArch64::X6 || Reg == AArch64::W6) && - !Subtarget->isXRegisterReserved(6)) || - ((Reg == AArch64::X7 || Reg == AArch64::W7) && - !Subtarget->isXRegisterReserved(7)) || - ((Reg == AArch64::X18 || Reg == AArch64::W18) && - !Subtarget->isXRegisterReserved(18)) || - ((Reg == AArch64::X20 || Reg == AArch64::W20) && - !Subtarget->isXRegisterReserved(20))) - Reg = 0; + unsigned Reg = MatchRegisterName(RegName); + if (AArch64::X1 <= Reg && Reg <= AArch64::X28) { + const MCRegisterInfo *MRI = Subtarget->getRegisterInfo(); + unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false); + if (!Subtarget->isXRegisterReserved(DwarfRegNum)) + Reg = 0; + } if (Reg) return Reg; report_fatal_error(Twine("Invalid register name \"" @@ -5398,35 +5472,41 @@ bool AArch64TargetLowering::isOffsetFoldingLegal( return false; } -bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { - // We can materialize #0.0 as fmov $Rd, XZR for 64-bit and 32-bit cases. - // FIXME: We should be able to handle f128 as well with a clever lowering. - if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32 || - (VT == MVT::f16 && Subtarget->hasFullFP16()))) { - LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString() << " imm value: 0\n"); - return true; - } - +bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, + bool OptForSize) const { bool IsLegal = false; - SmallString<128> ImmStrVal; - Imm.toString(ImmStrVal); - + // We can materialize #0.0 as fmov $Rd, XZR for 64-bit, 32-bit cases, and + // 16-bit case when target has full fp16 support. + // FIXME: We should be able to handle f128 as well with a clever lowering. + const APInt ImmInt = Imm.bitcastToAPInt(); if (VT == MVT::f64) - IsLegal = AArch64_AM::getFP64Imm(Imm) != -1; + IsLegal = AArch64_AM::getFP64Imm(ImmInt) != -1 || Imm.isPosZero(); else if (VT == MVT::f32) - IsLegal = AArch64_AM::getFP32Imm(Imm) != -1; + IsLegal = AArch64_AM::getFP32Imm(ImmInt) != -1 || Imm.isPosZero(); else if (VT == MVT::f16 && Subtarget->hasFullFP16()) - IsLegal = AArch64_AM::getFP16Imm(Imm) != -1; + IsLegal = AArch64_AM::getFP16Imm(ImmInt) != -1 || Imm.isPosZero(); + // TODO: fmov h0, w0 is also legal, however on't have an isel pattern to + // generate that fmov. - if (IsLegal) { - LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString() - << " imm value: " << ImmStrVal << "\n"); - return true; + // If we can not materialize in immediate field for fmov, check if the + // value can be encoded as the immediate operand of a logical instruction. + // The immediate value will be created with either MOVZ, MOVN, or ORR. + if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32)) { + // The cost is actually exactly the same for mov+fmov vs. adrp+ldr; + // however the mov+fmov sequence is always better because of the reduced + // cache pressure. The timings are still the same if you consider + // movw+movk+fmov vs. adrp+ldr (it's one instruction longer, but the + // movw+movk is fused). So we limit up to 2 instrdduction at most. + SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn; + AArch64_IMM::expandMOVImm(ImmInt.getZExtValue(), VT.getSizeInBits(), + Insn); + unsigned Limit = (OptForSize ? 1 : (Subtarget->hasFuseLiterals() ? 5 : 2)); + IsLegal = Insn.size() <= Limit; } - LLVM_DEBUG(dbgs() << "Illegal " << VT.getEVTString() - << " imm value: " << ImmStrVal << "\n"); - return false; + LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT.getEVTString() + << " imm value: "; Imm.dump();); + return IsLegal; } //===----------------------------------------------------------------------===// @@ -6226,6 +6306,8 @@ static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { unsigned NumElts = VT.getVectorNumElements(); + if (NumElts % 2 != 0) + return false; WhichResult = (M[0] == 0 ? 0 : 1); for (unsigned i = 0; i < NumElts; i += 2) { if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) || @@ -6240,6 +6322,8 @@ static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>. static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { unsigned NumElts = VT.getVectorNumElements(); + if (NumElts % 2 != 0) + return false; WhichResult = (M[0] == 0 ? 0 : 1); unsigned Idx = WhichResult * NumElts / 2; for (unsigned i = 0; i != NumElts; i += 2) { @@ -6276,6 +6360,8 @@ static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>. static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { unsigned NumElts = VT.getVectorNumElements(); + if (NumElts % 2 != 0) + return false; WhichResult = (M[0] == 0 ? 0 : 1); for (unsigned i = 0; i < NumElts; i += 2) { if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) || @@ -6918,46 +7004,6 @@ static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG, return SDValue(); } -SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op, - SelectionDAG &DAG) const { - SDValue LHS = Op.getOperand(0); - EVT VT = Op.getValueType(); - - BuildVectorSDNode *BVN = - dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode()); - if (!BVN) { - // AND commutes, so try swapping the operands. - LHS = Op.getOperand(1); - BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode()); - } - if (!BVN) - return Op; - - APInt DefBits(VT.getSizeInBits(), 0); - APInt UndefBits(VT.getSizeInBits(), 0); - if (resolveBuildVector(BVN, DefBits, UndefBits)) { - SDValue NewOp; - - // We only have BIC vector immediate instruction, which is and-not. - DefBits = ~DefBits; - if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG, - DefBits, &LHS)) || - (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG, - DefBits, &LHS))) - return NewOp; - - UndefBits = ~UndefBits; - if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG, - UndefBits, &LHS)) || - (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG, - UndefBits, &LHS))) - return NewOp; - } - - // We can always fall back to a non-immediate AND. - return Op; -} - // Specialized code to quickly find if PotentialBVec is a BuildVector that // consists of only the same constant int value, returned in reference arg // ConstVal @@ -7799,8 +7845,8 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op, // Make v4f16 (only) fcmp operations utilise vector instructions // v8f16 support will be a litle more complicated - if (LHS.getValueType().getVectorElementType() == MVT::f16) { - if (!FullFP16 && LHS.getValueType().getVectorNumElements() == 4) { + if (!FullFP16 && LHS.getValueType().getVectorElementType() == MVT::f16) { + if (LHS.getValueType().getVectorNumElements() == 4) { LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, LHS); RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, RHS); SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC); @@ -7810,8 +7856,8 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op, return SDValue(); } - assert(LHS.getValueType().getVectorElementType() == MVT::f32 || - LHS.getValueType().getVectorElementType() == MVT::f64); + assert((!FullFP16 && LHS.getValueType().getVectorElementType() != MVT::f16) || + LHS.getValueType().getVectorElementType() != MVT::f128); // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally // clean. Some of them require two branches to implement. @@ -8255,6 +8301,110 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const { return true; } +/// Check if both Op1 and Op2 are shufflevector extracts of either the lower +/// or upper half of the vector elements. +static bool areExtractShuffleVectors(Value *Op1, Value *Op2) { + auto areTypesHalfed = [](Value *FullV, Value *HalfV) { + auto *FullVT = cast<VectorType>(FullV->getType()); + auto *HalfVT = cast<VectorType>(HalfV->getType()); + return FullVT->getBitWidth() == 2 * HalfVT->getBitWidth(); + }; + + auto extractHalf = [](Value *FullV, Value *HalfV) { + auto *FullVT = cast<VectorType>(FullV->getType()); + auto *HalfVT = cast<VectorType>(HalfV->getType()); + return FullVT->getNumElements() == 2 * HalfVT->getNumElements(); + }; + + Constant *M1, *M2; + Value *S1Op1, *S2Op1; + if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) || + !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2)))) + return false; + + // Check that the operands are half as wide as the result and we extract + // half of the elements of the input vectors. + if (!areTypesHalfed(S1Op1, Op1) || !areTypesHalfed(S2Op1, Op2) || + !extractHalf(S1Op1, Op1) || !extractHalf(S2Op1, Op2)) + return false; + + // Check the mask extracts either the lower or upper half of vector + // elements. + int M1Start = -1; + int M2Start = -1; + int NumElements = cast<VectorType>(Op1->getType())->getNumElements() * 2; + if (!ShuffleVectorInst::isExtractSubvectorMask(M1, NumElements, M1Start) || + !ShuffleVectorInst::isExtractSubvectorMask(M2, NumElements, M2Start) || + M1Start != M2Start || (M1Start != 0 && M2Start != (NumElements / 2))) + return false; + + return true; +} + +/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth +/// of the vector elements. +static bool areExtractExts(Value *Ext1, Value *Ext2) { + auto areExtDoubled = [](Instruction *Ext) { + return Ext->getType()->getScalarSizeInBits() == + 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits(); + }; + + if (!match(Ext1, m_ZExtOrSExt(m_Value())) || + !match(Ext2, m_ZExtOrSExt(m_Value())) || + !areExtDoubled(cast<Instruction>(Ext1)) || + !areExtDoubled(cast<Instruction>(Ext2))) + return false; + + return true; +} + +/// Check if sinking \p I's operands to I's basic block is profitable, because +/// the operands can be folded into a target instruction, e.g. +/// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2). +bool AArch64TargetLowering::shouldSinkOperands( + Instruction *I, SmallVectorImpl<Use *> &Ops) const { + if (!I->getType()->isVectorTy()) + return false; + + if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { + switch (II->getIntrinsicID()) { + case Intrinsic::aarch64_neon_umull: + if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1))) + return false; + Ops.push_back(&II->getOperandUse(0)); + Ops.push_back(&II->getOperandUse(1)); + return true; + default: + return false; + } + } + + switch (I->getOpcode()) { + case Instruction::Sub: + case Instruction::Add: { + if (!areExtractExts(I->getOperand(0), I->getOperand(1))) + return false; + + // If the exts' operands extract either the lower or upper elements, we + // can sink them too. + auto Ext1 = cast<Instruction>(I->getOperand(0)); + auto Ext2 = cast<Instruction>(I->getOperand(1)); + if (areExtractShuffleVectors(Ext1, Ext2)) { + Ops.push_back(&Ext1->getOperandUse(0)); + Ops.push_back(&Ext2->getOperandUse(0)); + } + + Ops.push_back(&I->getOperandUse(0)); + Ops.push_back(&I->getOperandUse(1)); + + return true; + } + default: + return false; + } + return false; +} + bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType, unsigned &RequiredAligment) const { if (!LoadedType.isSimple() || @@ -8377,8 +8527,9 @@ bool AArch64TargetLowering::lowerInterleavedLoad( // If we're generating more than one load, compute the base address of // subsequent loads as an offset from the previous. if (LoadCount > 0) - BaseAddr = Builder.CreateConstGEP1_32( - BaseAddr, VecTy->getVectorNumElements() * Factor); + BaseAddr = + Builder.CreateConstGEP1_32(VecTy->getVectorElementType(), BaseAddr, + VecTy->getVectorNumElements() * Factor); CallInst *LdN = Builder.CreateCall( LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy), "ldN"); @@ -8540,7 +8691,8 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, // If we generating more than one store, we compute the base address of // subsequent stores as an offset from the previous. if (StoreCount > 0) - BaseAddr = Builder.CreateConstGEP1_32(BaseAddr, LaneLen * Factor); + BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getVectorElementType(), + BaseAddr, LaneLen * Factor); Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy)); Builder.CreateCall(StNFunc, Ops); @@ -8554,13 +8706,12 @@ static bool memOpAlign(unsigned DstAlign, unsigned SrcAlign, (DstAlign == 0 || DstAlign % AlignCheck == 0)); } -EVT AArch64TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned DstAlign, - unsigned SrcAlign, bool IsMemset, - bool ZeroMemset, - bool MemcpyStrSrc, - MachineFunction &MF) const { - const Function &F = MF.getFunction(); - bool CanImplicitFloat = !F.hasFnAttribute(Attribute::NoImplicitFloat); +EVT AArch64TargetLowering::getOptimalMemOpType( + uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset, + bool ZeroMemset, bool MemcpyStrSrc, + const AttributeList &FuncAttributes) const { + bool CanImplicitFloat = + !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat); bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat; bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat; // Only use AdvSIMD to implement memset of 32-byte and above. It would have @@ -8571,7 +8722,9 @@ EVT AArch64TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned DstAlign, if (memOpAlign(SrcAlign, DstAlign, AlignCheck)) return true; bool Fast; - return allowsMisalignedMemoryAccesses(VT, 0, 1, &Fast) && Fast; + return allowsMisalignedMemoryAccesses(VT, 0, 1, MachineMemOperand::MONone, + &Fast) && + Fast; }; if (CanUseNEON && IsMemset && !IsSmallMemset && @@ -9061,6 +9214,9 @@ static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG, if (!Subtarget->hasNEON()) return SDValue(); + if (!N->getValueType(0).isSimple()) + return SDValue(); + SDValue Op = N->getOperand(0); if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() || Op.getOpcode() != ISD::FMUL) @@ -9323,6 +9479,46 @@ static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, return SDValue(); } +static SDValue performANDCombine(SDNode *N, + TargetLowering::DAGCombinerInfo &DCI) { + SelectionDAG &DAG = DCI.DAG; + SDValue LHS = N->getOperand(0); + EVT VT = N->getValueType(0); + if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT)) + return SDValue(); + + BuildVectorSDNode *BVN = + dyn_cast<BuildVectorSDNode>(N->getOperand(1).getNode()); + if (!BVN) + return SDValue(); + + // AND does not accept an immediate, so check if we can use a BIC immediate + // instruction instead. We do this here instead of using a (and x, (mvni imm)) + // pattern in isel, because some immediates may be lowered to the preferred + // (and x, (movi imm)) form, even though an mvni representation also exists. + APInt DefBits(VT.getSizeInBits(), 0); + APInt UndefBits(VT.getSizeInBits(), 0); + if (resolveBuildVector(BVN, DefBits, UndefBits)) { + SDValue NewOp; + + DefBits = ~DefBits; + if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG, + DefBits, &LHS)) || + (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG, + DefBits, &LHS))) + return NewOp; + + UndefBits = ~UndefBits; + if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG, + UndefBits, &LHS)) || + (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG, + UndefBits, &LHS))) + return NewOp; + } + + return SDValue(); +} + static SDValue performSRLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { SelectionDAG &DAG = DCI.DAG; @@ -9598,12 +9794,13 @@ static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) { DAG.getConstant(NumElems, dl, MVT::i64)); } -static bool isEssentiallyExtractSubvector(SDValue N) { - if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) - return true; - - return N.getOpcode() == ISD::BITCAST && - N.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR; +static bool isEssentiallyExtractHighSubvector(SDValue N) { + if (N.getOpcode() == ISD::BITCAST) + N = N.getOperand(0); + if (N.getOpcode() != ISD::EXTRACT_SUBVECTOR) + return false; + return cast<ConstantSDNode>(N.getOperand(1))->getAPIntValue() == + N.getOperand(0).getValueType().getVectorNumElements() / 2; } /// Helper structure to keep track of ISD::SET_CC operands. @@ -9770,13 +9967,13 @@ static SDValue performAddSubLongCombine(SDNode *N, // It's not worth doing if at least one of the inputs isn't already an // extract, but we don't know which it'll be so we have to try both. - if (isEssentiallyExtractSubvector(LHS.getOperand(0))) { + if (isEssentiallyExtractHighSubvector(LHS.getOperand(0))) { RHS = tryExtendDUPToExtractHigh(RHS.getOperand(0), DAG); if (!RHS.getNode()) return SDValue(); RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS); - } else if (isEssentiallyExtractSubvector(RHS.getOperand(0))) { + } else if (isEssentiallyExtractHighSubvector(RHS.getOperand(0))) { LHS = tryExtendDUPToExtractHigh(LHS.getOperand(0), DAG); if (!LHS.getNode()) return SDValue(); @@ -9809,11 +10006,11 @@ static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N, // Either node could be a DUP, but it's not worth doing both of them (you'd // just as well use the non-high version) so look for a corresponding extract // operation on the other "wing". - if (isEssentiallyExtractSubvector(LHS)) { + if (isEssentiallyExtractHighSubvector(LHS)) { RHS = tryExtendDUPToExtractHigh(RHS, DAG); if (!RHS.getNode()) return SDValue(); - } else if (isEssentiallyExtractSubvector(RHS)) { + } else if (isEssentiallyExtractHighSubvector(RHS)) { LHS = tryExtendDUPToExtractHigh(LHS, DAG); if (!LHS.getNode()) return SDValue(); @@ -10261,7 +10458,7 @@ static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, return SDValue(); // Don't split at -Oz. - if (DAG.getMachineFunction().getFunction().optForMinSize()) + if (DAG.getMachineFunction().getFunction().hasMinSize()) return SDValue(); // Don't split v2i64 vectors. Memcpy lowering produces those and splitting @@ -10917,6 +11114,12 @@ static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert, return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); } + // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits. + if (Op->getOpcode() == ISD::ANY_EXTEND && + Bit < Op->getOperand(0).getValueSizeInBits()) { + return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); + } + if (Op->getNumOperands() != 2) return Op; @@ -11172,6 +11375,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, return performFDivCombine(N, DAG, DCI, Subtarget); case ISD::OR: return performORCombine(N, DCI, Subtarget); + case ISD::AND: + return performANDCombine(N, DCI); case ISD::SRL: return performSRLCombine(N, DCI); case ISD::INTRINSIC_WO_CHAIN: @@ -11573,6 +11778,9 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { // For the real atomic operations, we have ldxr/stxr up to 128 bits, TargetLowering::AtomicExpansionKind AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { + if (AI->isFloatingPointOperation()) + return AtomicExpansionKind::CmpXChg; + unsigned Size = AI->getType()->getPrimitiveSizeInBits(); if (Size > 128) return AtomicExpansionKind::None; // Nand not supported in LSE. @@ -11627,9 +11835,13 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr; Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys); - return Builder.CreateTruncOrBitCast( - Builder.CreateCall(Ldxr, Addr), - cast<PointerType>(Addr->getType())->getElementType()); + Type *EltTy = cast<PointerType>(Addr->getType())->getElementType(); + + const DataLayout &DL = M->getDataLayout(); + IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(EltTy)); + Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr, Addr), IntEltTy); + + return Builder.CreateBitCast(Trunc, EltTy); } void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance( @@ -11664,6 +11876,10 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder, Type *Tys[] = { Addr->getType() }; Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys); + const DataLayout &DL = M->getDataLayout(); + IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType())); + Val = Builder.CreateBitCast(Val, IntValTy); + return Builder.CreateCall(Stxr, {Builder.CreateZExtOrBitCast( Val, Stxr->getFunctionType()->getParamType(0)), @@ -11685,8 +11901,9 @@ static Value *UseTlsOffset(IRBuilder<> &IRB, unsigned Offset) { Function *ThreadPointerFunc = Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); return IRB.CreatePointerCast( - IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), Offset), - Type::getInt8PtrTy(IRB.getContext())->getPointerTo(0)); + IRB.CreateConstGEP1_32(IRB.getInt8Ty(), IRB.CreateCall(ThreadPointerFunc), + Offset), + IRB.getInt8PtrTy()->getPointerTo(0)); } Value *AArch64TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const { @@ -11712,12 +11929,13 @@ void AArch64TargetLowering::insertSSPDeclarations(Module &M) const { Type::getInt8PtrTy(M.getContext())); // MSVC CRT has a function to validate security cookie. - auto *SecurityCheckCookie = cast<Function>( - M.getOrInsertFunction("__security_check_cookie", - Type::getVoidTy(M.getContext()), - Type::getInt8PtrTy(M.getContext()))); - SecurityCheckCookie->setCallingConv(CallingConv::Win64); - SecurityCheckCookie->addAttribute(1, Attribute::AttrKind::InReg); + FunctionCallee SecurityCheckCookie = M.getOrInsertFunction( + "__security_check_cookie", Type::getVoidTy(M.getContext()), + Type::getInt8PtrTy(M.getContext())); + if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) { + F->setCallingConv(CallingConv::Win64); + F->addAttribute(1, Attribute::AttrKind::InReg); + } return; } TargetLowering::insertSSPDeclarations(M); @@ -11730,7 +11948,7 @@ Value *AArch64TargetLowering::getSDagStackGuard(const Module &M) const { return TargetLowering::getSDagStackGuard(M); } -Value *AArch64TargetLowering::getSSPStackGuardCheck(const Module &M) const { +Function *AArch64TargetLowering::getSSPStackGuardCheck(const Module &M) const { // MSVC CRT has a function to validate security cookie. if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment()) return M.getFunction("__security_check_cookie"); @@ -11825,6 +12043,11 @@ bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const { return OptSize && !VT.isVector(); } +bool AArch64TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { + // We want inc-of-add for scalars and sub-of-not for vectors. + return VT.isScalarInteger(); +} + bool AArch64TargetLowering::enableAggressiveFMAFusion(EVT VT) const { return Subtarget->hasAggressiveFMA() && VT.isFloatingPoint(); } diff --git a/lib/Target/AArch64/AArch64ISelLowering.h b/lib/Target/AArch64/AArch64ISelLowering.h index ffc4cc3ef534..4421c31f65c9 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.h +++ b/lib/Target/AArch64/AArch64ISelLowering.h @@ -1,9 +1,8 @@ //==-- AArch64ISelLowering.h - AArch64 DAG Lowering Interface ----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -215,7 +214,13 @@ enum NodeType : unsigned { LD4LANEpost, ST2LANEpost, ST3LANEpost, - ST4LANEpost + ST4LANEpost, + + STG, + STZG, + ST2G, + STZ2G + }; } // end namespace AArch64ISD @@ -263,9 +268,10 @@ public: /// Returns true if the target allows unaligned memory accesses of the /// specified type. - bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace = 0, - unsigned Align = 1, - bool *Fast = nullptr) const override; + bool allowsMisalignedMemoryAccesses( + EVT VT, unsigned AddrSpace = 0, unsigned Align = 1, + MachineMemOperand::Flags Flags = MachineMemOperand::MONone, + bool *Fast = nullptr) const override; /// Provide custom lowering hooks for some operations. SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; @@ -287,7 +293,8 @@ public: bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; - bool isFPImmLegal(const APFloat &Imm, EVT VT) const override; + bool isFPImmLegal(const APFloat &Imm, EVT VT, + bool ForCodeSize) const override; /// Return true if the given shuffle mask can be codegen'd directly, or if it /// should be stack expanded. @@ -328,6 +335,9 @@ public: bool isZExtFree(EVT VT1, EVT VT2) const override; bool isZExtFree(SDValue Val, EVT VT2) const override; + bool shouldSinkOperands(Instruction *I, + SmallVectorImpl<Use *> &Ops) const override; + bool hasPairedLoad(EVT LoadedType, unsigned &RequiredAligment) const override; unsigned getMaxSupportedInterleaveFactor() const override { return 4; } @@ -346,7 +356,7 @@ public: EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc, - MachineFunction &MF) const override; + const AttributeList &FuncAttributes) const override; /// Return true if the addressing mode represented by AM is legal for this /// target, for a load/store of the specified type. @@ -409,7 +419,7 @@ public: void insertSSPDeclarations(Module &M) const override; Value *getSDagStackGuard(const Module &M) const override; - Value *getSSPStackGuardCheck(const Module &M) const override; + Function *getSSPStackGuardCheck(const Module &M) const override; /// If the target has a standard location for the unsafe stack pointer, /// returns the address of that location. Otherwise, returns nullptr. @@ -470,6 +480,12 @@ public: return VT.getSizeInBits() >= 64; // vector 'bic' } + bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override { + if (DAG.getMachineFunction().getFunction().hasMinSize()) + return false; + return true; + } + bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override { // For vectors, we don't have a preference.. @@ -487,6 +503,8 @@ public: return VTIsOk(XVT) && VTIsOk(KeptBitsVT); } + bool preferIncOfAddToSubOfNot(EVT VT) const override; + bool hasBitPreservingFPLogic(EVT VT) const override { // FIXME: Is this always true? It should be true for vectors at least. return VT == MVT::f32 || VT == MVT::f64; @@ -648,9 +666,9 @@ private: SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerVectorAND(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const; diff --git a/lib/Target/AArch64/AArch64InstrAtomics.td b/lib/Target/AArch64/AArch64InstrAtomics.td index 35cd7735ceb7..e22cb44d81ae 100644 --- a/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/lib/Target/AArch64/AArch64InstrAtomics.td @@ -1,9 +1,8 @@ //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64InstrFormats.td b/lib/Target/AArch64/AArch64InstrFormats.td index 9061ed4f9f54..d619137b55c5 100644 --- a/lib/Target/AArch64/AArch64InstrFormats.td +++ b/lib/Target/AArch64/AArch64InstrFormats.td @@ -1,9 +1,8 @@ //===- AArch64InstrFormats.td - AArch64 Instruction Formats --*- tblgen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -356,6 +355,9 @@ def am_indexed7s32 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S32", []>; def am_indexed7s64 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S64", []>; def am_indexed7s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S128", []>; +def am_indexedu6s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedU6S128", []>; +def am_indexeds9s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedS9S128", []>; + // uimm5sN predicate - True if the immediate is a multiple of N in the range // [0 * N, 32 * N]. def UImm5s2Operand : UImmScaledMemoryIndexed<5, 2>; @@ -1818,6 +1820,14 @@ multiclass Shift<bits<2> shift_type, string asm, SDNode OpNode> { def : Pat<(i32 (OpNode GPR32:$Rn, (i64 (sext GPR32:$Rm)))), (!cast<Instruction>(NAME # "Wr") GPR32:$Rn, GPR32:$Rm)>; + + def : Pat<(i64 (OpNode GPR64:$Rn, (i64 (sext GPR32:$Rm)))), + (!cast<Instruction>(NAME # "Xr") GPR64:$Rn, + (SUBREG_TO_REG (i32 0), GPR32:$Rm, sub_32))>; + + def : Pat<(i64 (OpNode GPR64:$Rn, (i64 (zext GPR32:$Rm)))), + (!cast<Instruction>(NAME # "Xr") GPR64:$Rn, + (SUBREG_TO_REG (i32 0), GPR32:$Rm, sub_32))>; } class ShiftAlias<string asm, Instruction inst, RegisterClass regtype> @@ -2332,7 +2342,7 @@ class AddSubG<bit isSub, string asm_inst, SDPatternOperator OpNode> } class SUBP<bit setsFlags, string asm_instr, SDPatternOperator OpNode> - : BaseTwoOperand<0b0000, GPR64, asm_instr, null_frag, GPR64sp, GPR64sp> { + : BaseTwoOperand<0b0000, GPR64, asm_instr, OpNode, GPR64sp, GPR64sp> { let Inst{31} = 1; let Inst{29} = setsFlags; } @@ -4017,7 +4027,7 @@ class BaseMemTag<bits<2> opc1, bits<2> opc2, string asm_insn, class MemTagVector<bit Load, string asm_insn, string asm_opnds, dag oops, dag iops> : BaseMemTag<{0b1, Load}, 0b00, asm_insn, asm_opnds, - "$Rn = $wback,@earlyclobber $wback", oops, iops> { + "", oops, iops> { bits<5> Rt; let Inst{20-12} = 0b000000000; @@ -4027,8 +4037,9 @@ class MemTagVector<bit Load, string asm_insn, string asm_opnds, } class MemTagLoad<string asm_insn, string asm_opnds> - : BaseMemTag<0b01, 0b00, asm_insn, asm_opnds, "", (outs GPR64:$Rt), - (ins GPR64sp:$Rn, simm9s16:$offset)> { + : BaseMemTag<0b01, 0b00, asm_insn, asm_opnds, "$Rt = $wback", + (outs GPR64:$wback), + (ins GPR64:$Rt, GPR64sp:$Rn, simm9s16:$offset)> { bits<5> Rt; bits<9> offset; @@ -4045,29 +4056,28 @@ class BaseMemTagStore<bits<2> opc1, bits<2> opc2, string asm_insn, bits<9> offset; let Inst{20-12} = offset; - let Inst{4-0} = 0b11111; - let Unpredictable{4-0} = 0b11111; + let Inst{4-0} = Rt; let mayStore = 1; } multiclass MemTagStore<bits<2> opc1, string insn> { def Offset : - BaseMemTagStore<opc1, 0b10, insn, "\t[$Rn, $offset]", "", - (outs), (ins GPR64sp:$Rn, simm9s16:$offset)>; + BaseMemTagStore<opc1, 0b10, insn, "\t$Rt, [$Rn, $offset]", "", + (outs), (ins GPR64sp:$Rt, GPR64sp:$Rn, simm9s16:$offset)>; def PreIndex : - BaseMemTagStore<opc1, 0b11, insn, "\t[$Rn, $offset]!", - "$Rn = $wback,@earlyclobber $wback", + BaseMemTagStore<opc1, 0b11, insn, "\t$Rt, [$Rn, $offset]!", + "$Rn = $wback", (outs GPR64sp:$wback), - (ins GPR64sp:$Rn, simm9s16:$offset)>; + (ins GPR64sp:$Rt, GPR64sp:$Rn, simm9s16:$offset)>; def PostIndex : - BaseMemTagStore<opc1, 0b01, insn, "\t[$Rn], $offset", - "$Rn = $wback,@earlyclobber $wback", + BaseMemTagStore<opc1, 0b01, insn, "\t$Rt, [$Rn], $offset", + "$Rn = $wback", (outs GPR64sp:$wback), - (ins GPR64sp:$Rn, simm9s16:$offset)>; + (ins GPR64sp:$Rt, GPR64sp:$Rn, simm9s16:$offset)>; - def : InstAlias<insn # "\t[$Rn]", - (!cast<Instruction>(NAME # "Offset") GPR64sp:$Rn, 0)>; + def : InstAlias<insn # "\t$Rt, [$Rn]", + (!cast<Instruction>(NAME # "Offset") GPR64sp:$Rt, GPR64sp:$Rn, 0)>; } //--- diff --git a/lib/Target/AArch64/AArch64InstrInfo.cpp b/lib/Target/AArch64/AArch64InstrInfo.cpp index ada067888572..215e96a82d0e 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -1,9 +1,8 @@ //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -77,8 +76,11 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { const MachineFunction *MF = MBB.getParent(); const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo(); - if (MI.getOpcode() == AArch64::INLINEASM) - return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI); + { + auto Op = MI.getOpcode(); + if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR) + return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI); + } // FIXME: We currently only handle pseudoinstructions that don't get expanded // before the assembly printer. @@ -928,9 +930,9 @@ bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, } bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint( - MachineInstr &MIa, MachineInstr &MIb, AliasAnalysis *AA) const { + const MachineInstr &MIa, const MachineInstr &MIb, AliasAnalysis *AA) const { const TargetRegisterInfo *TRI = &getRegisterInfo(); - MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; + const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; int64_t OffsetA = 0, OffsetB = 0; unsigned WidthA = 0, WidthB = 0; @@ -1715,6 +1717,69 @@ bool AArch64InstrInfo::isUnscaledLdSt(unsigned Opc) { } } +Optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) { + switch (Opc) { + default: return {}; + case AArch64::PRFMui: return AArch64::PRFUMi; + case AArch64::LDRXui: return AArch64::LDURXi; + case AArch64::LDRWui: return AArch64::LDURWi; + case AArch64::LDRBui: return AArch64::LDURBi; + case AArch64::LDRHui: return AArch64::LDURHi; + case AArch64::LDRSui: return AArch64::LDURSi; + case AArch64::LDRDui: return AArch64::LDURDi; + case AArch64::LDRQui: return AArch64::LDURQi; + case AArch64::LDRBBui: return AArch64::LDURBBi; + case AArch64::LDRHHui: return AArch64::LDURHHi; + case AArch64::LDRSBXui: return AArch64::LDURSBXi; + case AArch64::LDRSBWui: return AArch64::LDURSBWi; + case AArch64::LDRSHXui: return AArch64::LDURSHXi; + case AArch64::LDRSHWui: return AArch64::LDURSHWi; + case AArch64::LDRSWui: return AArch64::LDURSWi; + case AArch64::STRXui: return AArch64::STURXi; + case AArch64::STRWui: return AArch64::STURWi; + case AArch64::STRBui: return AArch64::STURBi; + case AArch64::STRHui: return AArch64::STURHi; + case AArch64::STRSui: return AArch64::STURSi; + case AArch64::STRDui: return AArch64::STURDi; + case AArch64::STRQui: return AArch64::STURQi; + case AArch64::STRBBui: return AArch64::STURBBi; + case AArch64::STRHHui: return AArch64::STURHHi; + } +} + +unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) { + switch (Opc) { + default: + return 2; + case AArch64::LDPXi: + case AArch64::LDPDi: + case AArch64::STPXi: + case AArch64::STPDi: + case AArch64::LDNPXi: + case AArch64::LDNPDi: + case AArch64::STNPXi: + case AArch64::STNPDi: + case AArch64::LDPQi: + case AArch64::STPQi: + case AArch64::LDNPQi: + case AArch64::STNPQi: + case AArch64::LDPWi: + case AArch64::LDPSi: + case AArch64::STPWi: + case AArch64::STPSi: + case AArch64::LDNPWi: + case AArch64::LDNPSi: + case AArch64::STNPWi: + case AArch64::STNPSi: + case AArch64::LDG: + case AArch64::STGPi: + return 3; + case AArch64::ADDG: + case AArch64::STGOffset: + return 2; + } +} + bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) { switch (MI.getOpcode()) { default: @@ -1837,7 +1902,7 @@ unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc, // Is this a candidate for ld/st merging or pairing? For example, we don't // touch volatiles or load/stores that have a hint to avoid pair formation. -bool AArch64InstrInfo::isCandidateToMergeOrPair(MachineInstr &MI) const { +bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const { // If this is a volatile load/store, don't mess with it. if (MI.hasOrderedMemoryRef()) return false; @@ -1879,8 +1944,8 @@ bool AArch64InstrInfo::isCandidateToMergeOrPair(MachineInstr &MI) const { return true; } -bool AArch64InstrInfo::getMemOperandWithOffset(MachineInstr &LdSt, - MachineOperand *&BaseOp, +bool AArch64InstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, + const MachineOperand *&BaseOp, int64_t &Offset, const TargetRegisterInfo *TRI) const { unsigned Width; @@ -1888,7 +1953,7 @@ bool AArch64InstrInfo::getMemOperandWithOffset(MachineInstr &LdSt, } bool AArch64InstrInfo::getMemOperandWithOffsetWidth( - MachineInstr &LdSt, MachineOperand *&BaseOp, int64_t &Offset, + const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, unsigned &Width, const TargetRegisterInfo *TRI) const { assert(LdSt.mayLoadOrStore() && "Expected a memory operation."); // Handle only loads/stores with base register followed by immediate offset. @@ -1944,7 +2009,7 @@ AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const { bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale, unsigned &Width, int64_t &MinOffset, - int64_t &MaxOffset) const { + int64_t &MaxOffset) { switch (Opcode) { // Not a memory operation or something we want to handle. default: @@ -1965,6 +2030,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale, MinOffset = -256; MaxOffset = 255; break; + case AArch64::PRFUMi: case AArch64::LDURXi: case AArch64::LDURDi: case AArch64::STURXi: @@ -2034,6 +2100,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale, MinOffset = -64; MaxOffset = 63; break; + case AArch64::PRFMui: case AArch64::LDRXui: case AArch64::LDRDui: case AArch64::STRXui: @@ -2066,6 +2133,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale, break; case AArch64::LDRHui: case AArch64::LDRHHui: + case AArch64::LDRSHWui: + case AArch64::LDRSHXui: case AArch64::STRHui: case AArch64::STRHHui: Scale = Width = 2; @@ -2074,12 +2143,40 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale, break; case AArch64::LDRBui: case AArch64::LDRBBui: + case AArch64::LDRSBWui: + case AArch64::LDRSBXui: case AArch64::STRBui: case AArch64::STRBBui: Scale = Width = 1; MinOffset = 0; MaxOffset = 4095; break; + case AArch64::ADDG: + case AArch64::TAGPstack: + Scale = 16; + Width = 0; + MinOffset = 0; + MaxOffset = 63; + break; + case AArch64::LDG: + case AArch64::STGOffset: + case AArch64::STZGOffset: + Scale = Width = 16; + MinOffset = -256; + MaxOffset = 255; + break; + case AArch64::ST2GOffset: + case AArch64::STZ2GOffset: + Scale = 16; + Width = 32; + MinOffset = -256; + MaxOffset = 255; + break; + case AArch64::STGPi: + Scale = Width = 16; + MinOffset = -64; + MaxOffset = 63; + break; } return true; @@ -2181,11 +2278,11 @@ static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, /// Detect opportunities for ldp/stp formation. /// /// Only called for LdSt for which getMemOperandWithOffset returns true. -bool AArch64InstrInfo::shouldClusterMemOps(MachineOperand &BaseOp1, - MachineOperand &BaseOp2, +bool AArch64InstrInfo::shouldClusterMemOps(const MachineOperand &BaseOp1, + const MachineOperand &BaseOp2, unsigned NumLoads) const { - MachineInstr &FirstLdSt = *BaseOp1.getParent(); - MachineInstr &SecondLdSt = *BaseOp2.getParent(); + const MachineInstr &FirstLdSt = *BaseOp1.getParent(); + const MachineInstr &SecondLdSt = *BaseOp2.getParent(); if (BaseOp1.getType() != BaseOp2.getType()) return false; @@ -2292,6 +2389,31 @@ void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB, } } +void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB, + MachineBasicBlock::iterator I, + DebugLoc DL, unsigned DestReg, + unsigned SrcReg, bool KillSrc, + unsigned Opcode, unsigned ZeroReg, + llvm::ArrayRef<unsigned> Indices) const { + const TargetRegisterInfo *TRI = &getRegisterInfo(); + unsigned NumRegs = Indices.size(); + +#ifndef NDEBUG + uint16_t DestEncoding = TRI->getEncodingValue(DestReg); + uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg); + assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 && + "GPR reg sequences should not be able to overlap"); +#endif + + for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) { + const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode)); + AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI); + MIB.addReg(ZeroReg); + AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI); + MIB.addImm(0); + } +} + void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, unsigned DestReg, @@ -2431,6 +2553,22 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, return; } + if (AArch64::XSeqPairsClassRegClass.contains(DestReg) && + AArch64::XSeqPairsClassRegClass.contains(SrcReg)) { + static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64}; + copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs, + AArch64::XZR, Indices); + return; + } + + if (AArch64::WSeqPairsClassRegClass.contains(DestReg) && + AArch64::WSeqPairsClassRegClass.contains(SrcReg)) { + static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32}; + copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs, + AArch64::WZR, Indices); + return; + } + if (AArch64::FPR128RegClass.contains(DestReg) && AArch64::FPR128RegClass.contains(SrcReg)) { if (Subtarget.hasNEON()) { @@ -2839,7 +2977,7 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB, unsigned DestReg, unsigned SrcReg, int Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool SetNZCV, - bool NeedsWinCFI) { + bool NeedsWinCFI, bool *HasWinCFI) { if (DestReg == SrcReg && Offset == 0) return; @@ -2884,10 +3022,13 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB, .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftSize)) .setMIFlag(Flag); - if (NeedsWinCFI && SrcReg == AArch64::SP && DestReg == AArch64::SP) - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) - .addImm(ThisVal) - .setMIFlag(Flag); + if (NeedsWinCFI && SrcReg == AArch64::SP && DestReg == AArch64::SP) { + if (HasWinCFI) + *HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) + .addImm(ThisVal) + .setMIFlag(Flag); + } SrcReg = DestReg; Offset -= ThisVal; @@ -2903,6 +3044,8 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB, if (NeedsWinCFI) { if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) || (SrcReg == AArch64::FP && DestReg == AArch64::SP)) { + if (HasWinCFI) + *HasWinCFI = true; if (Offset == 0) BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)). setMIFlag(Flag); @@ -2910,6 +3053,8 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB, BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP)). addImm(Offset).setMIFlag(Flag); } else if (DestReg == AArch64::SP) { + if (HasWinCFI) + *HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)). addImm(Offset).setMIFlag(Flag); } @@ -2919,7 +3064,7 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB, MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl( MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, - LiveIntervals *LIS) const { + LiveIntervals *LIS, VirtRegMap *VRM) const { // This is a bit of a hack. Consider this instruction: // // %0 = COPY %sp; GPR64all:%0 @@ -3102,11 +3247,6 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset, bool *OutUseUnscaledOp, unsigned *OutUnscaledOp, int *EmittableOffset) { - int Scale = 1; - bool IsSigned = false; - // The ImmIdx should be changed case by case if it is not 2. - unsigned ImmIdx = 2; - unsigned UnscaledOp = 0; // Set output values in case of early exit. if (EmittableOffset) *EmittableOffset = 0; @@ -3114,10 +3254,12 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset, *OutUseUnscaledOp = false; if (OutUnscaledOp) *OutUnscaledOp = 0; + + // Exit early for structured vector spills/fills as they can't take an + // immediate offset. switch (MI.getOpcode()) { default: - llvm_unreachable("unhandled opcode in rewriteAArch64FrameIndex"); - // Vector spills/fills can't take an immediate offset. + break; case AArch64::LD1Twov2d: case AArch64::LD1Threev2d: case AArch64::LD1Fourv2d: @@ -3130,208 +3272,53 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset, case AArch64::ST1Twov1d: case AArch64::ST1Threev1d: case AArch64::ST1Fourv1d: + case AArch64::IRG: + case AArch64::IRGstack: return AArch64FrameOffsetCannotUpdate; - case AArch64::PRFMui: - Scale = 8; - UnscaledOp = AArch64::PRFUMi; - break; - case AArch64::LDRXui: - Scale = 8; - UnscaledOp = AArch64::LDURXi; - break; - case AArch64::LDRWui: - Scale = 4; - UnscaledOp = AArch64::LDURWi; - break; - case AArch64::LDRBui: - Scale = 1; - UnscaledOp = AArch64::LDURBi; - break; - case AArch64::LDRHui: - Scale = 2; - UnscaledOp = AArch64::LDURHi; - break; - case AArch64::LDRSui: - Scale = 4; - UnscaledOp = AArch64::LDURSi; - break; - case AArch64::LDRDui: - Scale = 8; - UnscaledOp = AArch64::LDURDi; - break; - case AArch64::LDRQui: - Scale = 16; - UnscaledOp = AArch64::LDURQi; - break; - case AArch64::LDRBBui: - Scale = 1; - UnscaledOp = AArch64::LDURBBi; - break; - case AArch64::LDRHHui: - Scale = 2; - UnscaledOp = AArch64::LDURHHi; - break; - case AArch64::LDRSBXui: - Scale = 1; - UnscaledOp = AArch64::LDURSBXi; - break; - case AArch64::LDRSBWui: - Scale = 1; - UnscaledOp = AArch64::LDURSBWi; - break; - case AArch64::LDRSHXui: - Scale = 2; - UnscaledOp = AArch64::LDURSHXi; - break; - case AArch64::LDRSHWui: - Scale = 2; - UnscaledOp = AArch64::LDURSHWi; - break; - case AArch64::LDRSWui: - Scale = 4; - UnscaledOp = AArch64::LDURSWi; - break; - - case AArch64::STRXui: - Scale = 8; - UnscaledOp = AArch64::STURXi; - break; - case AArch64::STRWui: - Scale = 4; - UnscaledOp = AArch64::STURWi; - break; - case AArch64::STRBui: - Scale = 1; - UnscaledOp = AArch64::STURBi; - break; - case AArch64::STRHui: - Scale = 2; - UnscaledOp = AArch64::STURHi; - break; - case AArch64::STRSui: - Scale = 4; - UnscaledOp = AArch64::STURSi; - break; - case AArch64::STRDui: - Scale = 8; - UnscaledOp = AArch64::STURDi; - break; - case AArch64::STRQui: - Scale = 16; - UnscaledOp = AArch64::STURQi; - break; - case AArch64::STRBBui: - Scale = 1; - UnscaledOp = AArch64::STURBBi; - break; - case AArch64::STRHHui: - Scale = 2; - UnscaledOp = AArch64::STURHHi; - break; - - case AArch64::LDPXi: - case AArch64::LDPDi: - case AArch64::STPXi: - case AArch64::STPDi: - case AArch64::LDNPXi: - case AArch64::LDNPDi: - case AArch64::STNPXi: - case AArch64::STNPDi: - ImmIdx = 3; - IsSigned = true; - Scale = 8; - break; - case AArch64::LDPQi: - case AArch64::STPQi: - case AArch64::LDNPQi: - case AArch64::STNPQi: - ImmIdx = 3; - IsSigned = true; - Scale = 16; - break; - case AArch64::LDPWi: - case AArch64::LDPSi: - case AArch64::STPWi: - case AArch64::STPSi: - case AArch64::LDNPWi: - case AArch64::LDNPSi: - case AArch64::STNPWi: - case AArch64::STNPSi: - ImmIdx = 3; - IsSigned = true; - Scale = 4; - break; - - case AArch64::LDURXi: - case AArch64::LDURWi: - case AArch64::LDURBi: - case AArch64::LDURHi: - case AArch64::LDURSi: - case AArch64::LDURDi: - case AArch64::LDURQi: - case AArch64::LDURHHi: - case AArch64::LDURBBi: - case AArch64::LDURSBXi: - case AArch64::LDURSBWi: - case AArch64::LDURSHXi: - case AArch64::LDURSHWi: - case AArch64::LDURSWi: - case AArch64::STURXi: - case AArch64::STURWi: - case AArch64::STURBi: - case AArch64::STURHi: - case AArch64::STURSi: - case AArch64::STURDi: - case AArch64::STURQi: - case AArch64::STURBBi: - case AArch64::STURHHi: - Scale = 1; - break; } - Offset += MI.getOperand(ImmIdx).getImm() * Scale; + // Get the min/max offset and the scale. + unsigned Scale, Width; + int64_t MinOff, MaxOff; + if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), Scale, Width, MinOff, + MaxOff)) + llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal"); + + // Construct the complete offset. + const MachineOperand &ImmOpnd = + MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode())); + Offset += ImmOpnd.getImm() * Scale; - bool useUnscaledOp = false; // If the offset doesn't match the scale, we rewrite the instruction to // use the unscaled instruction instead. Likewise, if we have a negative - // offset (and have an unscaled op to use). - if ((Offset & (Scale - 1)) != 0 || (Offset < 0 && UnscaledOp != 0)) - useUnscaledOp = true; + // offset and there is an unscaled op to use. + Optional<unsigned> UnscaledOp = + AArch64InstrInfo::getUnscaledLdSt(MI.getOpcode()); + bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0); + if (useUnscaledOp && + !AArch64InstrInfo::getMemOpInfo(*UnscaledOp, Scale, Width, MinOff, MaxOff)) + llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal"); - // Use an unscaled addressing mode if the instruction has a negative offset - // (or if the instruction is already using an unscaled addressing mode). - unsigned MaskBits; - if (IsSigned) { - // ldp/stp instructions. - MaskBits = 7; - Offset /= Scale; - } else if (UnscaledOp == 0 || useUnscaledOp) { - MaskBits = 9; - IsSigned = true; - Scale = 1; - } else { - MaskBits = 12; - IsSigned = false; - Offset /= Scale; - } + int64_t Remainder = Offset % Scale; + assert(!(Remainder && useUnscaledOp) && + "Cannot have remainder when using unscaled op"); - // Attempt to fold address computation. - int MaxOff = (1 << (MaskBits - IsSigned)) - 1; - int MinOff = (IsSigned ? (-MaxOff - 1) : 0); - if (Offset >= MinOff && Offset <= MaxOff) { - if (EmittableOffset) - *EmittableOffset = Offset; - Offset = 0; - } else { - int NewOff = Offset < 0 ? MinOff : MaxOff; - if (EmittableOffset) - *EmittableOffset = NewOff; - Offset = (Offset - NewOff) * Scale; + assert(MinOff < MaxOff && "Unexpected Min/Max offsets"); + int NewOffset = Offset / Scale; + if (MinOff <= NewOffset && NewOffset <= MaxOff) + Offset = Remainder; + else { + NewOffset = NewOffset < 0 ? MinOff : MaxOff; + Offset = Offset - NewOffset * Scale + Remainder; } + + if (EmittableOffset) + *EmittableOffset = NewOffset; if (OutUseUnscaledOp) *OutUseUnscaledOp = useUnscaledOp; - if (OutUnscaledOp) - *OutUnscaledOp = UnscaledOp; + if (OutUnscaledOp && UnscaledOp) + *OutUnscaledOp = *UnscaledOp; + return AArch64FrameOffsetCanUpdate | (Offset == 0 ? AArch64FrameOffsetIsLegal : 0); } @@ -4974,8 +4961,8 @@ AArch64InstrInfo::getOutliningCandidateInfo( // At this point, we have a stack instruction that we might need to // fix up. We'll handle it if it's a load or store. if (MI.mayLoadOrStore()) { - MachineOperand *Base; // Filled with the base operand of MI. - int64_t Offset; // Filled with the offset of MI. + const MachineOperand *Base; // Filled with the base operand of MI. + int64_t Offset; // Filled with the offset of MI. // Does it allow us to offset the base operand and is the base the // register SP? @@ -5331,12 +5318,20 @@ AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, MI.modifiesRegister(AArch64::W30, &getRegisterInfo())) return outliner::InstrType::Illegal; + // Don't outline BTI instructions, because that will prevent the outlining + // site from being indirectly callable. + if (MI.getOpcode() == AArch64::HINT) { + int64_t Imm = MI.getOperand(0).getImm(); + if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38) + return outliner::InstrType::Illegal; + } + return outliner::InstrType::Legal; } void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const { for (MachineInstr &MI : MBB) { - MachineOperand *Base; + const MachineOperand *Base; unsigned Width; int64_t Offset; @@ -5534,7 +5529,32 @@ MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall( bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault( MachineFunction &MF) const { - return MF.getFunction().optForMinSize(); + return MF.getFunction().hasMinSize(); +} + +bool AArch64InstrInfo::isCopyInstrImpl( + const MachineInstr &MI, const MachineOperand *&Source, + const MachineOperand *&Destination) const { + + // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg + // and zero immediate operands used as an alias for mov instruction. + if (MI.getOpcode() == AArch64::ORRWrs && + MI.getOperand(1).getReg() == AArch64::WZR && + MI.getOperand(3).getImm() == 0x0) { + Destination = &MI.getOperand(0); + Source = &MI.getOperand(2); + return true; + } + + if (MI.getOpcode() == AArch64::ORRXrs && + MI.getOperand(1).getReg() == AArch64::XZR && + MI.getOperand(3).getImm() == 0x0) { + Destination = &MI.getOperand(0); + Source = &MI.getOperand(2); + return true; + } + + return false; } #define GET_INSTRINFO_HELPERS diff --git a/lib/Target/AArch64/AArch64InstrInfo.h b/lib/Target/AArch64/AArch64InstrInfo.h index 9954669d5675..7be4daba7dc4 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.h +++ b/lib/Target/AArch64/AArch64InstrInfo.h @@ -1,9 +1,8 @@ //===- AArch64InstrInfo.h - AArch64 Instruction Information -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #include "AArch64.h" #include "AArch64RegisterInfo.h" +#include "llvm/ADT/Optional.h" #include "llvm/CodeGen/MachineCombinerPattern.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -54,7 +54,8 @@ public: unsigned &DstReg, unsigned &SubIdx) const override; bool - areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb, + areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, + const MachineInstr &MIb, AliasAnalysis *AA = nullptr) const override; unsigned isLoadFromStackSlot(const MachineInstr &MI, @@ -84,6 +85,14 @@ public: return isUnscaledLdSt(MI.getOpcode()); } + /// Returns the unscaled load/store for the scaled load/store opcode, + /// if there is a corresponding unscaled variant available. + static Optional<unsigned> getUnscaledLdSt(unsigned Opc); + + + /// Returns the index for the immediate for a given instruction. + static unsigned getLoadStoreImmIdx(unsigned Opc); + /// Return true if pairing the given load or store may be paired with another. static bool isPairableLdStInst(const MachineInstr &MI); @@ -92,16 +101,18 @@ public: static unsigned convertToFlagSettingOpc(unsigned Opc, bool &Is64Bit); /// Return true if this is a load/store that can be potentially paired/merged. - bool isCandidateToMergeOrPair(MachineInstr &MI) const; + bool isCandidateToMergeOrPair(const MachineInstr &MI) const; /// Hint that pairing the given load or store is unprofitable. static void suppressLdStPair(MachineInstr &MI); - bool getMemOperandWithOffset(MachineInstr &MI, MachineOperand *&BaseOp, + bool getMemOperandWithOffset(const MachineInstr &MI, + const MachineOperand *&BaseOp, int64_t &Offset, const TargetRegisterInfo *TRI) const override; - bool getMemOperandWithOffsetWidth(MachineInstr &MI, MachineOperand *&BaseOp, + bool getMemOperandWithOffsetWidth(const MachineInstr &MI, + const MachineOperand *&BaseOp, int64_t &Offset, unsigned &Width, const TargetRegisterInfo *TRI) const; @@ -112,16 +123,21 @@ public: /// \p Scale, \p Width, \p MinOffset, and \p MaxOffset accordingly. /// /// For unscaled instructions, \p Scale is set to 1. - bool getMemOpInfo(unsigned Opcode, unsigned &Scale, unsigned &Width, - int64_t &MinOffset, int64_t &MaxOffset) const; + static bool getMemOpInfo(unsigned Opcode, unsigned &Scale, unsigned &Width, + int64_t &MinOffset, int64_t &MaxOffset); - bool shouldClusterMemOps(MachineOperand &BaseOp1, MachineOperand &BaseOp2, + bool shouldClusterMemOps(const MachineOperand &BaseOp1, + const MachineOperand &BaseOp2, unsigned NumLoads) const override; void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef<unsigned> Indices) const; + void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, + DebugLoc DL, unsigned DestReg, unsigned SrcReg, + bool KillSrc, unsigned Opcode, unsigned ZeroReg, + llvm::ArrayRef<unsigned> Indices) const; void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, bool KillSrc) const override; @@ -146,7 +162,8 @@ public: foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, - LiveIntervals *LIS = nullptr) const override; + LiveIntervals *LIS = nullptr, + VirtRegMap *VRM = nullptr) const override; /// \returns true if a branch from an instruction with opcode \p BranchOpc /// bytes is capable of jumping to a position \p BrOffset bytes away. @@ -251,6 +268,13 @@ public: #define GET_INSTRINFO_HELPER_DECLS #include "AArch64GenInstrInfo.inc" +protected: + /// If the specific machine instruction is a instruction that moves/copies + /// value from one register to another register return true along with + /// @Source machine operand and @Destination machine operand. + bool isCopyInstrImpl(const MachineInstr &MI, const MachineOperand *&Source, + const MachineOperand *&Destination) const override; + private: /// Sets the offsets on outlined instructions in \p MBB which use SP /// so that they will be valid post-outlining. @@ -277,7 +301,8 @@ void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag = MachineInstr::NoFlags, - bool SetNZCV = false, bool NeedsWinCFI = false); + bool SetNZCV = false, bool NeedsWinCFI = false, + bool *HasWinCFI = nullptr); /// rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the /// FP. Return false if the offset could not be handled directly in MI, and diff --git a/lib/Target/AArch64/AArch64InstrInfo.td b/lib/Target/AArch64/AArch64InstrInfo.td index c24b8b36441b..eed53f36d574 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.td +++ b/lib/Target/AArch64/AArch64InstrInfo.td @@ -1,9 +1,8 @@ //=- AArch64InstrInfo.td - Describe the AArch64 Instructions -*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -108,6 +107,16 @@ def HasFuseAES : Predicate<"Subtarget->hasFuseAES()">, "fuse-aes">; def HasSVE : Predicate<"Subtarget->hasSVE()">, AssemblerPredicate<"FeatureSVE", "sve">; +def HasSVE2 : Predicate<"Subtarget->hasSVE2()">, + AssemblerPredicate<"FeatureSVE2", "sve2">; +def HasSVE2AES : Predicate<"Subtarget->hasSVE2AES()">, + AssemblerPredicate<"FeatureSVE2AES", "sve2-aes">; +def HasSVE2SM4 : Predicate<"Subtarget->hasSVE2SM4()">, + AssemblerPredicate<"FeatureSVE2SM4", "sve2-sm4">; +def HasSVE2SHA3 : Predicate<"Subtarget->hasSVE2SHA3()">, + AssemblerPredicate<"FeatureSVE2SHA3", "sve2-sha3">; +def HasSVE2BitPerm : Predicate<"Subtarget->hasSVE2BitPerm()">, + AssemblerPredicate<"FeatureSVE2BitPerm", "bitperm">; def HasRCPC : Predicate<"Subtarget->hasRCPC()">, AssemblerPredicate<"FeatureRCPC", "rcpc">; def HasAltNZCV : Predicate<"Subtarget->hasAlternativeNZCV()">, @@ -126,6 +135,7 @@ def HasMTE : Predicate<"Subtarget->hasMTE()">, AssemblerPredicate<"FeatureMTE", "mte">; def IsLE : Predicate<"Subtarget->isLittleEndian()">; def IsBE : Predicate<"!Subtarget->isLittleEndian()">; +def IsWindows : Predicate<"Subtarget->isTargetWindows()">; def UseAlternateSExtLoadCVTF32 : Predicate<"Subtarget->useAlternateSExtLoadCVTF32Pattern()">; @@ -133,6 +143,10 @@ def UseNegativeImmediates : Predicate<"false">, AssemblerPredicate<"!FeatureNoNegativeImmediates", "NegativeImmediates">; +def AArch64LocalRecover : SDNode<"ISD::LOCAL_RECOVER", + SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, + SDTCisInt<1>]>>; + //===----------------------------------------------------------------------===// // AArch64-specific DAG Nodes. @@ -395,6 +409,12 @@ def AArch64uminv : SDNode<"AArch64ISD::UMINV", SDT_AArch64UnaryVec>; def AArch64smaxv : SDNode<"AArch64ISD::SMAXV", SDT_AArch64UnaryVec>; def AArch64umaxv : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>; +def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>; +def AArch64stg : SDNode<"AArch64ISD::STG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; +def AArch64stzg : SDNode<"AArch64ISD::STZG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; +def AArch64st2g : SDNode<"AArch64ISD::ST2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; +def AArch64stz2g : SDNode<"AArch64ISD::STZ2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; + //===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===// @@ -404,10 +424,10 @@ def AArch64umaxv : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>; // the Function object through the <Target>Subtarget and objections were raised // to that (see post-commit review comments for r301750). let RecomputePerFunction = 1 in { - def ForCodeSize : Predicate<"MF->getFunction().optForSize()">; - def NotForCodeSize : Predicate<"!MF->getFunction().optForSize()">; + def ForCodeSize : Predicate<"MF->getFunction().hasOptSize()">; + def NotForCodeSize : Predicate<"!MF->getFunction().hasOptSize()">; // Avoid generating STRQro if it is slow, unless we're optimizing for code size. - def UseSTRQro : Predicate<"!Subtarget->isSTRQroSlow() || MF->getFunction().optForSize()">; + def UseSTRQro : Predicate<"!Subtarget->isSTRQroSlow() || MF->getFunction().hasOptSize()">; def UseBTI : Predicate<[{ MF->getFunction().hasFnAttribute("branch-target-enforcement") }]>; def NotUseBTI : Predicate<[{ !MF->getFunction().hasFnAttribute("branch-target-enforcement") }]>; @@ -703,7 +723,9 @@ let Predicates = [HasPA] in { // v8.3a floating point conversion for javascript let Predicates = [HasJS, HasFPARMv8] in def FJCVTZS : BaseFPToIntegerUnscaled<0b01, 0b11, 0b110, FPR64, GPR32, - "fjcvtzs", []> { + "fjcvtzs", + [(set GPR32:$Rd, + (int_aarch64_fjcvtzs FPR64:$Rn))]> { let Inst{31} = 0; } // HasJS, HasFPARMv8 @@ -760,6 +782,13 @@ def MSRpstateImm4 : MSRpstateImm0_15; def MOVbaseTLS : Pseudo<(outs GPR64:$dst), (ins), [(set GPR64:$dst, AArch64threadpointer)]>, Sched<[WriteSys]>; +let Uses = [ X9 ], Defs = [ X16, X17, LR, NZCV ] in { +def HWASAN_CHECK_MEMACCESS : Pseudo< + (outs), (ins GPR64noip:$ptr, i32imm:$accessinfo), + [(int_hwasan_check_memaccess X9, GPR64noip:$ptr, (i32 imm:$accessinfo))]>, + Sched<[]>; +} + // The cycle counter PMC register is PMCCNTR_EL0. let Predicates = [HasPerfMon] in def : Pat<(readcyclecounter), (MRS 0xdce8)>; @@ -1223,11 +1252,11 @@ defm : STOPregister<"stumin","LDUMIN">;// STUMINx // v8.5 Memory Tagging Extension let Predicates = [HasMTE] in { -def IRG : BaseTwoOperand<0b0100, GPR64sp, "irg", null_frag, GPR64sp, GPR64>, +def IRG : BaseTwoOperand<0b0100, GPR64sp, "irg", int_aarch64_irg, GPR64sp, GPR64>, Sched<[]>{ let Inst{31} = 1; } -def GMI : BaseTwoOperand<0b0101, GPR64, "gmi", null_frag, GPR64sp>, Sched<[]>{ +def GMI : BaseTwoOperand<0b0101, GPR64, "gmi", int_aarch64_gmi, GPR64sp>, Sched<[]>{ let Inst{31} = 1; let isNotDuplicable = 1; } @@ -1236,7 +1265,7 @@ def SUBG : AddSubG<1, "subg", null_frag>; def : InstAlias<"irg $dst, $src", (IRG GPR64sp:$dst, GPR64sp:$src, XZR), 1>; -def SUBP : SUBP<0, "subp", null_frag>, Sched<[]>; +def SUBP : SUBP<0, "subp", int_aarch64_subp>, Sched<[]>; def SUBPS : SUBP<1, "subps", null_frag>, Sched<[]>{ let Defs = [NZCV]; } @@ -1244,24 +1273,74 @@ def SUBPS : SUBP<1, "subps", null_frag>, Sched<[]>{ def : InstAlias<"cmpp $lhs, $rhs", (SUBPS XZR, GPR64sp:$lhs, GPR64sp:$rhs), 0>; def LDG : MemTagLoad<"ldg", "\t$Rt, [$Rn, $offset]">; + +def : Pat<(int_aarch64_addg (am_indexedu6s128 GPR64sp:$Rn, uimm6s16:$imm6), imm0_15:$imm4), + (ADDG GPR64sp:$Rn, imm0_63:$imm6, imm0_15:$imm4)>; +def : Pat<(int_aarch64_ldg GPR64:$Rt, (am_indexeds9s128 GPR64sp:$Rn, simm9s16:$offset)), + (LDG GPR64:$Rt, GPR64sp:$Rn, simm9s16:$offset)>; + def : InstAlias<"ldg $Rt, [$Rn]", (LDG GPR64:$Rt, GPR64sp:$Rn, 0), 1>; -def LDGV : MemTagVector<1, "ldgv", "\t$Rt, [$Rn]!", - (outs GPR64sp:$wback, GPR64:$Rt), (ins GPR64sp:$Rn)> { - let DecoderMethod = "DecodeLoadAllocTagArrayInstruction"; +def LDGM : MemTagVector<1, "ldgm", "\t$Rt, [$Rn]", + (outs GPR64:$Rt), (ins GPR64sp:$Rn)>; +def STGM : MemTagVector<0, "stgm", "\t$Rt, [$Rn]", + (outs), (ins GPR64:$Rt, GPR64sp:$Rn)>; +def STZGM : MemTagVector<0, "stzgm", "\t$Rt, [$Rn]", + (outs), (ins GPR64:$Rt, GPR64sp:$Rn)> { + let Inst{23} = 0; } -def STGV : MemTagVector<0, "stgv", "\t$Rt, [$Rn]!", - (outs GPR64sp:$wback), (ins GPR64:$Rt, GPR64sp:$Rn)>; defm STG : MemTagStore<0b00, "stg">; defm STZG : MemTagStore<0b01, "stzg">; defm ST2G : MemTagStore<0b10, "st2g">; defm STZ2G : MemTagStore<0b11, "stz2g">; +def : Pat<(AArch64stg GPR64sp:$Rn, (am_indexeds9s128 GPR64sp:$Rm, simm9s16:$imm)), + (STGOffset $Rn, $Rm, $imm)>; +def : Pat<(AArch64stzg GPR64sp:$Rn, (am_indexeds9s128 GPR64sp:$Rm, simm9s16:$imm)), + (STZGOffset $Rn, $Rm, $imm)>; +def : Pat<(AArch64st2g GPR64sp:$Rn, (am_indexeds9s128 GPR64sp:$Rm, simm9s16:$imm)), + (ST2GOffset $Rn, $Rm, $imm)>; +def : Pat<(AArch64stz2g GPR64sp:$Rn, (am_indexeds9s128 GPR64sp:$Rm, simm9s16:$imm)), + (STZ2GOffset $Rn, $Rm, $imm)>; + defm STGP : StorePairOffset <0b01, 0, GPR64z, simm7s16, "stgp">; def STGPpre : StorePairPreIdx <0b01, 0, GPR64z, simm7s16, "stgp">; def STGPpost : StorePairPostIdx<0b01, 0, GPR64z, simm7s16, "stgp">; +def : Pat<(int_aarch64_stg GPR64:$Rt, (am_indexeds9s128 GPR64sp:$Rn, simm9s16:$offset)), + (STGOffset GPR64:$Rt, GPR64sp:$Rn, simm9s16:$offset)>; + +def : Pat<(int_aarch64_stgp (am_indexed7s128 GPR64sp:$Rn, simm7s16:$imm), GPR64:$Rt, GPR64:$Rt2), + (STGPi $Rt, $Rt2, $Rn, $imm)>; + +def IRGstack + : Pseudo<(outs GPR64sp:$Rd), (ins GPR64sp:$Rsp, GPR64:$Rm), []>, + Sched<[]>; +def TAGPstack + : Pseudo<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, uimm6s16:$imm6, GPR64sp:$Rm, imm0_15:$imm4), []>, + Sched<[]>; + +// Explicit SP in the first operand prevents ShrinkWrap optimization +// from leaving this instruction out of the stack frame. When IRGstack +// is transformed into IRG, this operand is replaced with the actual +// register / expression for the tagged base pointer of the current function. +def : Pat<(int_aarch64_irg_sp i64:$Rm), (IRGstack SP, i64:$Rm)>; + +// Large STG to be expanded into a loop. $Rm is the size, $Rn is start address. +// $Rn_wback is one past the end of the range. +let isCodeGenOnly=1, mayStore=1 in { +def STGloop + : Pseudo<(outs GPR64common:$Rm_wback, GPR64sp:$Rn_wback), (ins GPR64common:$Rm, GPR64sp:$Rn), + [], "$Rn = $Rn_wback,@earlyclobber $Rn_wback,$Rm = $Rm_wback,@earlyclobber $Rm_wback" >, + Sched<[WriteAdr, WriteST]>; + +def STZGloop + : Pseudo<(outs GPR64common:$Rm_wback, GPR64sp:$Rn_wback), (ins GPR64common:$Rm, GPR64sp:$Rn), + [], "$Rn = $Rn_wback,@earlyclobber $Rn_wback,$Rm = $Rm_wback,@earlyclobber $Rm_wback" >, + Sched<[WriteAdr, WriteST]>; +} + } // Predicates = [HasMTE] //===----------------------------------------------------------------------===// @@ -3052,6 +3131,27 @@ defm : FPToIntegerPats<fp_to_uint, ftrunc, "FCVTZU">; defm : FPToIntegerPats<fp_to_sint, fround, "FCVTAS">; defm : FPToIntegerPats<fp_to_uint, fround, "FCVTAU">; +let Predicates = [HasFullFP16] in { + def : Pat<(i32 (lround f16:$Rn)), + (!cast<Instruction>(FCVTASUWHr) f16:$Rn)>; + def : Pat<(i64 (lround f16:$Rn)), + (!cast<Instruction>(FCVTASUXHr) f16:$Rn)>; + def : Pat<(i64 (llround f16:$Rn)), + (!cast<Instruction>(FCVTASUXHr) f16:$Rn)>; +} +def : Pat<(i32 (lround f32:$Rn)), + (!cast<Instruction>(FCVTASUWSr) f32:$Rn)>; +def : Pat<(i32 (lround f64:$Rn)), + (!cast<Instruction>(FCVTASUWDr) f64:$Rn)>; +def : Pat<(i64 (lround f32:$Rn)), + (!cast<Instruction>(FCVTASUXSr) f32:$Rn)>; +def : Pat<(i64 (lround f64:$Rn)), + (!cast<Instruction>(FCVTASUXDr) f64:$Rn)>; +def : Pat<(i64 (llround f32:$Rn)), + (!cast<Instruction>(FCVTASUXSr) f32:$Rn)>; +def : Pat<(i64 (llround f64:$Rn)), + (!cast<Instruction>(FCVTASUXDr) f64:$Rn)>; + //===----------------------------------------------------------------------===// // Scaled integer to floating point conversion instructions. //===----------------------------------------------------------------------===// @@ -3116,6 +3216,27 @@ let Predicates = [HasFRInt3264] in { defm FRINT64X : FRIntNNT<0b11, "frint64x">; } // HasFRInt3264 +let Predicates = [HasFullFP16] in { + def : Pat<(i32 (lrint f16:$Rn)), + (FCVTZSUWHr (!cast<Instruction>(FRINTXHr) f16:$Rn))>; + def : Pat<(i64 (lrint f16:$Rn)), + (FCVTZSUXHr (!cast<Instruction>(FRINTXHr) f16:$Rn))>; + def : Pat<(i64 (llrint f16:$Rn)), + (FCVTZSUXHr (!cast<Instruction>(FRINTXHr) f16:$Rn))>; +} +def : Pat<(i32 (lrint f32:$Rn)), + (FCVTZSUWSr (!cast<Instruction>(FRINTXSr) f32:$Rn))>; +def : Pat<(i32 (lrint f64:$Rn)), + (FCVTZSUWDr (!cast<Instruction>(FRINTXDr) f64:$Rn))>; +def : Pat<(i64 (lrint f32:$Rn)), + (FCVTZSUXSr (!cast<Instruction>(FRINTXSr) f32:$Rn))>; +def : Pat<(i64 (lrint f64:$Rn)), + (FCVTZSUXDr (!cast<Instruction>(FRINTXDr) f64:$Rn))>; +def : Pat<(i64 (llrint f32:$Rn)), + (FCVTZSUXSr (!cast<Instruction>(FRINTXSr) f32:$Rn))>; +def : Pat<(i64 (llrint f64:$Rn)), + (FCVTZSUXDr (!cast<Instruction>(FRINTXDr) f64:$Rn))>; + //===----------------------------------------------------------------------===// // Floating point two operand instructions. //===----------------------------------------------------------------------===// @@ -3489,7 +3610,7 @@ def : Pat<(fabs (fsub VT:$Rn, VT:$Rm)), (!cast<Instruction>("FABD"#VT) VT:$Rn, V } defm FACGE : SIMDThreeSameVectorFPCmp<1,0,0b101,"facge",int_aarch64_neon_facge>; defm FACGT : SIMDThreeSameVectorFPCmp<1,1,0b101,"facgt",int_aarch64_neon_facgt>; -defm FADDP : SIMDThreeSameVectorFP<1,0,0b010,"faddp",int_aarch64_neon_addp>; +defm FADDP : SIMDThreeSameVectorFP<1,0,0b010,"faddp",int_aarch64_neon_faddp>; defm FADD : SIMDThreeSameVectorFP<0,0,0b010,"fadd", fadd>; defm FCMEQ : SIMDThreeSameVectorFPCmp<0, 0, 0b100, "fcmeq", AArch64fcmeq>; defm FCMGE : SIMDThreeSameVectorFPCmp<1, 0, 0b100, "fcmge", AArch64fcmge>; @@ -5314,6 +5435,8 @@ def : Pat<(f16 (int_aarch64_neon_vcvtfxs2fp (i32 (sext_inreg FPR32:$Rn, i16)), v (SCVTFh (EXTRACT_SUBREG FPR32:$Rn, hsub), vecshiftR16:$imm)>; def : Pat<(f16 (int_aarch64_neon_vcvtfxs2fp (i32 FPR32:$Rn), vecshiftR16:$imm)), (SCVTFh (EXTRACT_SUBREG FPR32:$Rn, hsub), vecshiftR16:$imm)>; +def : Pat<(f16 (int_aarch64_neon_vcvtfxs2fp (i64 FPR64:$Rn), vecshiftR16:$imm)), + (SCVTFh (EXTRACT_SUBREG FPR64:$Rn, hsub), vecshiftR16:$imm)>; def : Pat<(f16 (int_aarch64_neon_vcvtfxu2fp (and FPR32:$Rn, (i32 65535)), vecshiftR16:$imm)), @@ -5342,6 +5465,16 @@ def : Pat<(i64 (int_aarch64_neon_vcvtfp2fxu (f16 FPR16:$Rn), vecshiftR64:$imm)), (i64 (IMPLICIT_DEF)), (FCVTZUh FPR16:$Rn, vecshiftR64:$imm), hsub))>; +def : Pat<(i32 (int_aarch64_neon_facge (f16 FPR16:$Rn), (f16 FPR16:$Rm))), + (i32 (INSERT_SUBREG + (i32 (IMPLICIT_DEF)), + (FACGE16 FPR16:$Rn, FPR16:$Rm), + hsub))>; +def : Pat<(i32 (int_aarch64_neon_facgt (f16 FPR16:$Rn), (f16 FPR16:$Rm))), + (i32 (INSERT_SUBREG + (i32 (IMPLICIT_DEF)), + (FACGT16 FPR16:$Rn, FPR16:$Rm), + hsub))>; defm SHL : SIMDScalarLShiftD< 0, 0b01010, "shl", AArch64vshl>; defm SLI : SIMDScalarLShiftDTied<1, 0b01010, "sli">; @@ -6031,6 +6164,7 @@ def : Pat<(i32 (trunc GPR64sp:$src)), // __builtin_trap() uses the BRK instruction on AArch64. def : Pat<(trap), (BRK 1)>; +def : Pat<(debugtrap), (BRK 0xF000)>, Requires<[IsWindows]>; // Multiply high patterns which multiply the lower subvector using smull/umull // and the upper subvector with smull2/umull2. Then shuffle the high the high @@ -6147,6 +6281,7 @@ def : Pat<(v4i16 (AArch64NvCast (v2f32 FPR64:$src))), (v4i16 FPR64:$src)>; def : Pat<(v2i32 (AArch64NvCast (v2f32 FPR64:$src))), (v2i32 FPR64:$src)>; def : Pat<(v2f32 (AArch64NvCast (v2f32 FPR64:$src))), (v2f32 FPR64:$src)>; def : Pat<(v1i64 (AArch64NvCast (v2f32 FPR64:$src))), (v1i64 FPR64:$src)>; +def : Pat<(v1f64 (AArch64NvCast (v2f32 FPR64:$src))), (v1f64 FPR64:$src)>; // Natural vector casts (128 bit) def : Pat<(v16i8 (AArch64NvCast (v4i32 FPR128:$src))), (v16i8 FPR128:$src)>; @@ -6801,5 +6936,8 @@ def : Pat<(AArch64tcret tglobaladdr:$dst, (i32 timm:$FPDiff)), def : Pat<(AArch64tcret texternalsym:$dst, (i32 timm:$FPDiff)), (TCRETURNdi texternalsym:$dst, imm:$FPDiff)>; +def MOVMCSym : Pseudo<(outs GPR64:$dst), (ins i64imm:$sym), []>, Sched<[]>; +def : Pat<(i64 (AArch64LocalRecover mcsym:$sym)), (MOVMCSym mcsym:$sym)>; + include "AArch64InstrAtomics.td" include "AArch64SVEInstrInfo.td" diff --git a/lib/Target/AArch64/AArch64InstructionSelector.cpp b/lib/Target/AArch64/AArch64InstructionSelector.cpp index 5eb589bf66d5..4e13fb8e2027 100644 --- a/lib/Target/AArch64/AArch64InstructionSelector.cpp +++ b/lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -1,9 +1,8 @@ //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -19,11 +18,14 @@ #include "AArch64Subtarget.h" #include "AArch64TargetMachine.h" #include "MCTargetDesc/AArch64AddressingModes.h" +#include "llvm/ADT/Optional.h" #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -57,6 +59,15 @@ private: /// the patterns that don't require complex C++. bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; + // A lowering phase that runs before any selection attempts. + + void preISelLower(MachineInstr &I) const; + + // An early selection function that runs before the selectImpl() call. + bool earlySelect(MachineInstr &I) const; + + bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const; bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF, @@ -65,15 +76,84 @@ private: bool selectCompareBranch(MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const; + bool selectVectorASHR(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI) const; + // Helper to generate an equivalent of scalar_to_vector into a new register, // returned via 'Dst'. - bool emitScalarToVector(unsigned &Dst, const LLT DstTy, - const TargetRegisterClass *DstRC, unsigned Scalar, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineRegisterInfo &MRI) const; + MachineInstr *emitScalarToVector(unsigned EltSize, + const TargetRegisterClass *DstRC, + Register Scalar, + MachineIRBuilder &MIRBuilder) const; + + /// Emit a lane insert into \p DstReg, or a new vector register if None is + /// provided. + /// + /// The lane inserted into is defined by \p LaneIdx. The vector source + /// register is given by \p SrcReg. The register containing the element is + /// given by \p EltReg. + MachineInstr *emitLaneInsert(Optional<Register> DstReg, Register SrcReg, + Register EltReg, unsigned LaneIdx, + const RegisterBank &RB, + MachineIRBuilder &MIRBuilder) const; + bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI) const; bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const; bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const; + + void collectShuffleMaskIndices(MachineInstr &I, MachineRegisterInfo &MRI, + SmallVectorImpl<Optional<int>> &Idxs) const; + bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectSplitVectorUnmerge(MachineInstr &I, + MachineRegisterInfo &MRI) const; + bool selectIntrinsicWithSideEffects(MachineInstr &I, + MachineRegisterInfo &MRI) const; + bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI) const; + + unsigned emitConstantPoolEntry(Constant *CPVal, MachineFunction &MF) const; + MachineInstr *emitLoadFromConstantPool(Constant *CPVal, + MachineIRBuilder &MIRBuilder) const; + + // Emit a vector concat operation. + MachineInstr *emitVectorConcat(Optional<Register> Dst, Register Op1, + Register Op2, + MachineIRBuilder &MIRBuilder) const; + MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS, + MachineOperand &Predicate, + MachineIRBuilder &MIRBuilder) const; + MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS, + MachineIRBuilder &MIRBuilder) const; + MachineInstr *emitTST(const Register &LHS, const Register &RHS, + MachineIRBuilder &MIRBuilder) const; + MachineInstr *emitExtractVectorElt(Optional<Register> DstReg, + const RegisterBank &DstRB, LLT ScalarTy, + Register VecReg, unsigned LaneIdx, + MachineIRBuilder &MIRBuilder) const; + + /// Helper function for selecting G_FCONSTANT. If the G_FCONSTANT can be + /// materialized using a FMOV instruction, then update MI and return it. + /// Otherwise, do nothing and return a nullptr. + MachineInstr *emitFMovForFConstant(MachineInstr &MI, + MachineRegisterInfo &MRI) const; + + /// Emit a CSet for a compare. + MachineInstr *emitCSetForICMP(Register DefReg, unsigned Pred, + MachineIRBuilder &MIRBuilder) const; + + // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td. + // We use these manually instead of using the importer since it doesn't + // support SDNodeXForm. + ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const; + ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const; + ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const; + ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const; ComplexRendererFns selectArithImmed(MachineOperand &Root) const; @@ -109,6 +189,14 @@ private: void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned char OpFlags) const; + // Optimization methods. + bool tryOptVectorShuffle(MachineInstr &I) const; + bool tryOptVectorDup(MachineInstr &MI) const; + bool tryOptSelect(MachineInstr &MI) const; + MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS, + MachineOperand &Predicate, + MachineIRBuilder &MIRBuilder) const; + const AArch64TargetMachine &TM; const AArch64Subtarget &STI; const AArch64InstrInfo &TII; @@ -177,6 +265,70 @@ getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB, return nullptr; } +/// Given a register bank, and size in bits, return the smallest register class +/// that can represent that combination. +static const TargetRegisterClass * +getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits, + bool GetAllRegSet = false) { + unsigned RegBankID = RB.getID(); + + if (RegBankID == AArch64::GPRRegBankID) { + if (SizeInBits <= 32) + return GetAllRegSet ? &AArch64::GPR32allRegClass + : &AArch64::GPR32RegClass; + if (SizeInBits == 64) + return GetAllRegSet ? &AArch64::GPR64allRegClass + : &AArch64::GPR64RegClass; + } + + if (RegBankID == AArch64::FPRRegBankID) { + switch (SizeInBits) { + default: + return nullptr; + case 8: + return &AArch64::FPR8RegClass; + case 16: + return &AArch64::FPR16RegClass; + case 32: + return &AArch64::FPR32RegClass; + case 64: + return &AArch64::FPR64RegClass; + case 128: + return &AArch64::FPR128RegClass; + } + } + + return nullptr; +} + +/// Returns the correct subregister to use for a given register class. +static bool getSubRegForClass(const TargetRegisterClass *RC, + const TargetRegisterInfo &TRI, unsigned &SubReg) { + switch (TRI.getRegSizeInBits(*RC)) { + case 8: + SubReg = AArch64::bsub; + break; + case 16: + SubReg = AArch64::hsub; + break; + case 32: + if (RC == &AArch64::GPR32RegClass) + SubReg = AArch64::sub_32; + else + SubReg = AArch64::ssub; + break; + case 64: + SubReg = AArch64::dsub; + break; + default: + LLVM_DEBUG( + dbgs() << "Couldn't find appropriate subregister for register class."); + return false; + } + + return true; +} + /// Check whether \p I is a currently unsupported binary operation: /// - it has an unsized type /// - an operand is not a vreg @@ -332,107 +484,209 @@ static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, return GenericOpc; } -static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII, - MachineRegisterInfo &MRI, unsigned SrcReg) { - // Copies from gpr32 to fpr16 need to use a sub-register copy. - unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); - BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY)) - .addDef(CopyReg) - .addUse(SrcReg); - unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass); - BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) - .addDef(SubRegCopy) - .addUse(CopyReg, 0, AArch64::hsub); +#ifndef NDEBUG +/// Helper function that verifies that we have a valid copy at the end of +/// selectCopy. Verifies that the source and dest have the expected sizes and +/// then returns true. +static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank, + const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI, + const RegisterBankInfo &RBI) { + const unsigned DstReg = I.getOperand(0).getReg(); + const unsigned SrcReg = I.getOperand(1).getReg(); + const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); + const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); + + // Make sure the size of the source and dest line up. + assert( + (DstSize == SrcSize || + // Copies are a mean to setup initial types, the number of + // bits may not exactly match. + (TargetRegisterInfo::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || + // Copies are a mean to copy bits around, as long as we are + // on the same register class, that's fine. Otherwise, that + // means we need some SUBREG_TO_REG or AND & co. + (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && + "Copy with different width?!"); + + // Check the size of the destination. + assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && + "GPRs cannot get more than 64-bit width values"); + return true; +} +#endif + +/// Helper function for selectCopy. Inserts a subregister copy from +/// \p *From to \p *To, linking it up to \p I. +/// +/// e.g, given I = "Dst = COPY SrcReg", we'll transform that into +/// +/// CopyReg (From class) = COPY SrcReg +/// SubRegCopy (To class) = COPY CopyReg:SubReg +/// Dst = COPY SubRegCopy +static bool selectSubregisterCopy(MachineInstr &I, MachineRegisterInfo &MRI, + const RegisterBankInfo &RBI, unsigned SrcReg, + const TargetRegisterClass *From, + const TargetRegisterClass *To, + unsigned SubReg) { + MachineIRBuilder MIB(I); + auto Copy = MIB.buildCopy({From}, {SrcReg}); + auto SubRegCopy = MIB.buildInstr(TargetOpcode::COPY, {To}, {}) + .addReg(Copy.getReg(0), 0, SubReg); MachineOperand &RegOp = I.getOperand(1); - RegOp.setReg(SubRegCopy); + RegOp.setReg(SubRegCopy.getReg(0)); + + // It's possible that the destination register won't be constrained. Make + // sure that happens. + if (!TargetRegisterInfo::isPhysicalRegister(I.getOperand(0).getReg())) + RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI); + return true; } +/// Helper function to get the source and destination register classes for a +/// copy. Returns a std::pair containing the source register class for the +/// copy, and the destination register class for the copy. If a register class +/// cannot be determined, then it will be nullptr. +static std::pair<const TargetRegisterClass *, const TargetRegisterClass *> +getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII, + MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, + const RegisterBankInfo &RBI) { + unsigned DstReg = I.getOperand(0).getReg(); + unsigned SrcReg = I.getOperand(1).getReg(); + const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); + const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); + unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); + unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); + + // Special casing for cross-bank copies of s1s. We can technically represent + // a 1-bit value with any size of register. The minimum size for a GPR is 32 + // bits. So, we need to put the FPR on 32 bits as well. + // + // FIXME: I'm not sure if this case holds true outside of copies. If it does, + // then we can pull it into the helpers that get the appropriate class for a + // register bank. Or make a new helper that carries along some constraint + // information. + if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1)) + SrcSize = DstSize = 32; + + return {getMinClassForRegBank(SrcRegBank, SrcSize, true), + getMinClassForRegBank(DstRegBank, DstSize, true)}; +} + static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) { unsigned DstReg = I.getOperand(0).getReg(); unsigned SrcReg = I.getOperand(1).getReg(); + const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); + const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); - if (TargetRegisterInfo::isPhysicalRegister(DstReg)) { - if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) && - !TargetRegisterInfo::isPhysicalRegister(SrcReg)) { - const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI); - const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank( - MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true); - if (SrcRC == &AArch64::GPR32allRegClass) - return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg); - } - assert(I.isCopy() && "Generic operators do not allow physical registers"); - return true; - } - - const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); - const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); - (void)DstSize; - const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); - (void)SrcSize; - assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) && - "No phys reg on generic operators"); - assert( - (DstSize == SrcSize || - // Copies are a mean to setup initial types, the number of - // bits may not exactly match. - (TargetRegisterInfo::isPhysicalRegister(SrcReg) && - DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) || - // Copies are a mean to copy bits around, as long as we are - // on the same register class, that's fine. Otherwise, that - // means we need some SUBREG_TO_REG or AND & co. - (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && - "Copy with different width?!"); - assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) && - "GPRs cannot get more than 64-bit width values"); + // Find the correct register classes for the source and destination registers. + const TargetRegisterClass *SrcRC; + const TargetRegisterClass *DstRC; + std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI); - const TargetRegisterClass *RC = getRegClassForTypeOnBank( - MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true); - if (!RC) { - LLVM_DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n'); + if (!DstRC) { + LLVM_DEBUG(dbgs() << "Unexpected dest size " + << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'); return false; } - if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) { - const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg); - const TargetRegisterClass *SrcRC = - RegClassOrBank.dyn_cast<const TargetRegisterClass *>(); - const RegisterBank *RB = nullptr; + // A couple helpers below, for making sure that the copy we produce is valid. + + // Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want + // to verify that the src and dst are the same size, since that's handled by + // the SUBREG_TO_REG. + bool KnownValid = false; + + // Returns true, or asserts if something we don't expect happens. Instead of + // returning true, we return isValidCopy() to ensure that we verify the + // result. + auto CheckCopy = [&]() { + // If we have a bitcast or something, we can't have physical registers. + assert( + (I.isCopy() || + (!TargetRegisterInfo::isPhysicalRegister(I.getOperand(0).getReg()) && + !TargetRegisterInfo::isPhysicalRegister(I.getOperand(1).getReg()))) && + "No phys reg on generic operator!"); + assert(KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI)); + (void)KnownValid; + return true; + }; + + // Is this a copy? If so, then we may need to insert a subregister copy, or + // a SUBREG_TO_REG. + if (I.isCopy()) { + // Yes. Check if there's anything to fix up. if (!SrcRC) { - RB = RegClassOrBank.get<const RegisterBank *>(); - SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true); + LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n"); + return false; } - // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG. - if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) { - unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); - BuildMI(*I.getParent(), I, I.getDebugLoc(), - TII.get(AArch64::SUBREG_TO_REG)) - .addDef(PromoteReg) - .addImm(0) - .addUse(SrcReg) - .addImm(AArch64::hsub); - MachineOperand &RegOp = I.getOperand(1); - RegOp.setReg(PromoteReg); - } else if (RC == &AArch64::FPR16RegClass && - SrcRC == &AArch64::GPR32allRegClass) { - selectFP16CopyFromGPR32(I, TII, MRI, SrcReg); + + // Is this a cross-bank copy? + if (DstRegBank.getID() != SrcRegBank.getID()) { + // If we're doing a cross-bank copy on different-sized registers, we need + // to do a bit more work. + unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC); + unsigned DstSize = TRI.getRegSizeInBits(*DstRC); + + if (SrcSize > DstSize) { + // We're doing a cross-bank copy into a smaller register. We need a + // subregister copy. First, get a register class that's on the same bank + // as the destination, but the same size as the source. + const TargetRegisterClass *SubregRC = + getMinClassForRegBank(DstRegBank, SrcSize, true); + assert(SubregRC && "Didn't get a register class for subreg?"); + + // Get the appropriate subregister for the destination. + unsigned SubReg = 0; + if (!getSubRegForClass(DstRC, TRI, SubReg)) { + LLVM_DEBUG(dbgs() << "Couldn't determine subregister for copy.\n"); + return false; + } + + // Now, insert a subregister copy using the new register class. + selectSubregisterCopy(I, MRI, RBI, SrcReg, SubregRC, DstRC, SubReg); + return CheckCopy(); + } + + else if (DstRegBank.getID() == AArch64::GPRRegBankID && DstSize == 32 && + SrcSize == 16) { + // Special case for FPR16 to GPR32. + // FIXME: This can probably be generalized like the above case. + unsigned PromoteReg = + MRI.createVirtualRegister(&AArch64::FPR32RegClass); + BuildMI(*I.getParent(), I, I.getDebugLoc(), + TII.get(AArch64::SUBREG_TO_REG), PromoteReg) + .addImm(0) + .addUse(SrcReg) + .addImm(AArch64::hsub); + MachineOperand &RegOp = I.getOperand(1); + RegOp.setReg(PromoteReg); + + // Promise that the copy is implicitly validated by the SUBREG_TO_REG. + KnownValid = true; + } } + + // If the destination is a physical register, then there's nothing to + // change, so we're done. + if (TargetRegisterInfo::isPhysicalRegister(DstReg)) + return CheckCopy(); } - // No need to constrain SrcReg. It will get constrained when - // we hit another of its use or its defs. - // Copies do not have constraints. - if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { + // No need to constrain SrcReg. It will get constrained when we hit another + // of its use or its defs. Copies do not have constraints. + if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) << " operand\n"); return false; } I.setDesc(TII.get(AArch64::COPY)); - return true; + return CheckCopy(); } static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) { @@ -511,6 +765,46 @@ static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) { return GenericOpc; } +static unsigned selectSelectOpc(MachineInstr &I, MachineRegisterInfo &MRI, + const RegisterBankInfo &RBI) { + const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + bool IsFP = (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() != + AArch64::GPRRegBankID); + LLT Ty = MRI.getType(I.getOperand(0).getReg()); + if (Ty == LLT::scalar(32)) + return IsFP ? AArch64::FCSELSrrr : AArch64::CSELWr; + else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) + return IsFP ? AArch64::FCSELDrrr : AArch64::CSELXr; + return 0; +} + +/// Helper function to select the opcode for a G_FCMP. +static unsigned selectFCMPOpc(MachineInstr &I, MachineRegisterInfo &MRI) { + // If this is a compare against +0.0, then we don't have to explicitly + // materialize a constant. + const ConstantFP *FPImm = getConstantFPVRegVal(I.getOperand(3).getReg(), MRI); + bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative()); + unsigned OpSize = MRI.getType(I.getOperand(2).getReg()).getSizeInBits(); + if (OpSize != 32 && OpSize != 64) + return 0; + unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr}, + {AArch64::FCMPSri, AArch64::FCMPDri}}; + return CmpOpcTbl[ShouldUseImm][OpSize == 64]; +} + +/// Returns true if \p P is an unsigned integer comparison predicate. +static bool isUnsignedICMPPred(const CmpInst::Predicate P) { + switch (P) { + default: + return false; + case CmpInst::ICMP_UGT: + case CmpInst::ICMP_UGE: + case CmpInst::ICMP_ULT: + case CmpInst::ICMP_ULE: + return true; + } +} + static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) { switch (P) { default: @@ -595,7 +889,7 @@ static void changeFCMPPredToAArch64CC(CmpInst::Predicate P, bool AArch64InstructionSelector::selectCompareBranch( MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { - const unsigned CondReg = I.getOperand(0).getReg(); + const Register CondReg = I.getOperand(0).getReg(); MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); MachineInstr *CCMI = MRI.getVRegDef(CondReg); if (CCMI->getOpcode() == TargetOpcode::G_TRUNC) @@ -603,14 +897,25 @@ bool AArch64InstructionSelector::selectCompareBranch( if (CCMI->getOpcode() != TargetOpcode::G_ICMP) return false; - unsigned LHS = CCMI->getOperand(2).getReg(); - unsigned RHS = CCMI->getOperand(3).getReg(); - if (!getConstantVRegVal(RHS, MRI)) + Register LHS = CCMI->getOperand(2).getReg(); + Register RHS = CCMI->getOperand(3).getReg(); + auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI); + if (!VRegAndVal) std::swap(RHS, LHS); - const auto RHSImm = getConstantVRegVal(RHS, MRI); - if (!RHSImm || *RHSImm != 0) - return false; + VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI); + if (!VRegAndVal || VRegAndVal->Value != 0) { + MachineIRBuilder MIB(I); + // If we can't select a CBZ then emit a cmp + Bcc. + if (!emitIntegerCompare(CCMI->getOperand(2), CCMI->getOperand(3), + CCMI->getOperand(1), MIB)) + return false; + const AArch64CC::CondCode CC = changeICMPPredToAArch64CC( + (CmpInst::Predicate)CCMI->getOperand(1).getPredicate()); + MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB); + I.eraseFromParent(); + return true; + } const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI); if (RB.getID() != AArch64::GPRRegBankID) @@ -638,6 +943,74 @@ bool AArch64InstructionSelector::selectCompareBranch( return true; } +bool AArch64InstructionSelector::selectVectorSHL( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_SHL); + Register DstReg = I.getOperand(0).getReg(); + const LLT Ty = MRI.getType(DstReg); + Register Src1Reg = I.getOperand(1).getReg(); + Register Src2Reg = I.getOperand(2).getReg(); + + if (!Ty.isVector()) + return false; + + unsigned Opc = 0; + if (Ty == LLT::vector(4, 32)) { + Opc = AArch64::USHLv4i32; + } else if (Ty == LLT::vector(2, 32)) { + Opc = AArch64::USHLv2i32; + } else { + LLVM_DEBUG(dbgs() << "Unhandled G_SHL type"); + return false; + } + + MachineIRBuilder MIB(I); + auto UShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Src2Reg}); + constrainSelectedInstRegOperands(*UShl, TII, TRI, RBI); + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::selectVectorASHR( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_ASHR); + Register DstReg = I.getOperand(0).getReg(); + const LLT Ty = MRI.getType(DstReg); + Register Src1Reg = I.getOperand(1).getReg(); + Register Src2Reg = I.getOperand(2).getReg(); + + if (!Ty.isVector()) + return false; + + // There is not a shift right register instruction, but the shift left + // register instruction takes a signed value, where negative numbers specify a + // right shift. + + unsigned Opc = 0; + unsigned NegOpc = 0; + const TargetRegisterClass *RC = nullptr; + if (Ty == LLT::vector(4, 32)) { + Opc = AArch64::SSHLv4i32; + NegOpc = AArch64::NEGv4i32; + RC = &AArch64::FPR128RegClass; + } else if (Ty == LLT::vector(2, 32)) { + Opc = AArch64::SSHLv2i32; + NegOpc = AArch64::NEGv2i32; + RC = &AArch64::FPR64RegClass; + } else { + LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type"); + return false; + } + + MachineIRBuilder MIB(I); + auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg}); + constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI); + auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg}); + constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI); + I.eraseFromParent(); + return true; +} + bool AArch64InstructionSelector::selectVaStartAAPCS( MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { return false; @@ -646,9 +1019,9 @@ bool AArch64InstructionSelector::selectVaStartAAPCS( bool AArch64InstructionSelector::selectVaStartDarwin( MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); - unsigned ListReg = I.getOperand(0).getReg(); + Register ListReg = I.getOperand(0).getReg(); - unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); + Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri)) @@ -684,9 +1057,9 @@ void AArch64InstructionSelector::materializeLargeCMVal( MovZ->addOperand(MF, MachineOperand::CreateImm(0)); constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI); - auto BuildMovK = [&](unsigned SrcReg, unsigned char Flags, unsigned Offset, - unsigned ForceDstReg) { - unsigned DstReg = ForceDstReg + auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset, + Register ForceDstReg) { + Register DstReg = ForceDstReg ? ForceDstReg : MRI.createVirtualRegister(&AArch64::GPR64RegClass); auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg); @@ -702,13 +1075,105 @@ void AArch64InstructionSelector::materializeLargeCMVal( constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI); return DstReg; }; - unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(), + Register DstReg = BuildMovK(MovZ.getReg(0), AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0); DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0); BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg()); return; } +void AArch64InstructionSelector::preISelLower(MachineInstr &I) const { + MachineBasicBlock &MBB = *I.getParent(); + MachineFunction &MF = *MBB.getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + + switch (I.getOpcode()) { + case TargetOpcode::G_SHL: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: { + // These shifts are legalized to have 64 bit shift amounts because we want + // to take advantage of the existing imported selection patterns that assume + // the immediates are s64s. However, if the shifted type is 32 bits and for + // some reason we receive input GMIR that has an s64 shift amount that's not + // a G_CONSTANT, insert a truncate so that we can still select the s32 + // register-register variant. + unsigned SrcReg = I.getOperand(1).getReg(); + unsigned ShiftReg = I.getOperand(2).getReg(); + const LLT ShiftTy = MRI.getType(ShiftReg); + const LLT SrcTy = MRI.getType(SrcReg); + if (SrcTy.isVector()) + return; + assert(!ShiftTy.isVector() && "unexpected vector shift ty"); + if (SrcTy.getSizeInBits() != 32 || ShiftTy.getSizeInBits() != 64) + return; + auto *AmtMI = MRI.getVRegDef(ShiftReg); + assert(AmtMI && "could not find a vreg definition for shift amount"); + if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) { + // Insert a subregister copy to implement a 64->32 trunc + MachineIRBuilder MIB(I); + auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {}) + .addReg(ShiftReg, 0, AArch64::sub_32); + MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID)); + I.getOperand(2).setReg(Trunc.getReg(0)); + } + return; + } + default: + return; + } +} + +bool AArch64InstructionSelector::earlySelectSHL( + MachineInstr &I, MachineRegisterInfo &MRI) const { + // We try to match the immediate variant of LSL, which is actually an alias + // for a special case of UBFM. Otherwise, we fall back to the imported + // selector which will match the register variant. + assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op"); + const auto &MO = I.getOperand(2); + auto VRegAndVal = getConstantVRegVal(MO.getReg(), MRI); + if (!VRegAndVal) + return false; + + const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); + if (DstTy.isVector()) + return false; + bool Is64Bit = DstTy.getSizeInBits() == 64; + auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO); + auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO); + MachineIRBuilder MIB(I); + + if (!Imm1Fn || !Imm2Fn) + return false; + + auto NewI = + MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri, + {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()}); + + for (auto &RenderFn : *Imm1Fn) + RenderFn(NewI); + for (auto &RenderFn : *Imm2Fn) + RenderFn(NewI); + + I.eraseFromParent(); + return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI); +} + +bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const { + assert(I.getParent() && "Instruction should be in a basic block!"); + assert(I.getParent()->getParent() && "Instruction should be in a function!"); + + MachineBasicBlock &MBB = *I.getParent(); + MachineFunction &MF = *MBB.getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + + switch (I.getOpcode()) { + case TargetOpcode::G_SHL: + return earlySelectSHL(I, MRI); + default: + return false; + } +} + bool AArch64InstructionSelector::select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const { assert(I.getParent() && "Instruction should be in a basic block!"); @@ -727,30 +1192,27 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) { - const unsigned DefReg = I.getOperand(0).getReg(); + const Register DefReg = I.getOperand(0).getReg(); const LLT DefTy = MRI.getType(DefReg); - const TargetRegisterClass *DefRC = nullptr; - if (TargetRegisterInfo::isPhysicalRegister(DefReg)) { - DefRC = TRI.getRegClass(DefReg); - } else { - const RegClassOrRegBank &RegClassOrBank = - MRI.getRegClassOrRegBank(DefReg); + const RegClassOrRegBank &RegClassOrBank = + MRI.getRegClassOrRegBank(DefReg); - DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>(); + const TargetRegisterClass *DefRC + = RegClassOrBank.dyn_cast<const TargetRegisterClass *>(); + if (!DefRC) { + if (!DefTy.isValid()) { + LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n"); + return false; + } + const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>(); + DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI); if (!DefRC) { - if (!DefTy.isValid()) { - LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n"); - return false; - } - const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>(); - DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI); - if (!DefRC) { - LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n"); - return false; - } + LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n"); + return false; } } + I.setDesc(TII.get(TargetOpcode::PHI)); return RBI.constrainGenericRegister(DefReg, *DefRC, MRI); @@ -769,12 +1231,27 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return false; } + // Try to do some lowering before we start instruction selecting. These + // lowerings are purely transformations on the input G_MIR and so selection + // must continue after any modification of the instruction. + preISelLower(I); + + // There may be patterns where the importer can't deal with them optimally, + // but does select it to a suboptimal sequence so our custom C++ selection + // code later never has a chance to work on it. Therefore, we have an early + // selection attempt here to give priority to certain selection routines + // over the imported ones. + if (earlySelect(I)) + return true; + if (selectImpl(I, CoverageInfo)) return true; LLT Ty = I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{}; + MachineIRBuilder MIB(I); + switch (Opcode) { case TargetOpcode::G_BRCOND: { if (Ty.getSizeInBits() > 32) { @@ -786,7 +1263,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return false; } - const unsigned CondReg = I.getOperand(0).getReg(); + const Register CondReg = I.getOperand(0).getReg(); MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z @@ -826,15 +1303,57 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } + case TargetOpcode::G_BRJT: + return selectBrJT(I, MRI); + + case TargetOpcode::G_BSWAP: { + // Handle vector types for G_BSWAP directly. + Register DstReg = I.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + + // We should only get vector types here; everything else is handled by the + // importer right now. + if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) { + LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n"); + return false; + } + + // Only handle 4 and 2 element vectors for now. + // TODO: 16-bit elements. + unsigned NumElts = DstTy.getNumElements(); + if (NumElts != 4 && NumElts != 2) { + LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n"); + return false; + } + + // Choose the correct opcode for the supported types. Right now, that's + // v2s32, v4s32, and v2s64. + unsigned Opc = 0; + unsigned EltSize = DstTy.getElementType().getSizeInBits(); + if (EltSize == 32) + Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8 + : AArch64::REV32v16i8; + else if (EltSize == 64) + Opc = AArch64::REV64v16i8; + + // We should always get something by the time we get here... + assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?"); + + I.setDesc(TII.get(Opc)); + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); + } + case TargetOpcode::G_FCONSTANT: case TargetOpcode::G_CONSTANT: { const bool isFP = Opcode == TargetOpcode::G_FCONSTANT; + const LLT s8 = LLT::scalar(8); + const LLT s16 = LLT::scalar(16); const LLT s32 = LLT::scalar(32); const LLT s64 = LLT::scalar(64); const LLT p0 = LLT::pointer(0, 64); - const unsigned DefReg = I.getOperand(0).getReg(); + const Register DefReg = I.getOperand(0).getReg(); const LLT DefTy = MRI.getType(DefReg); const unsigned DefSize = DefTy.getSizeInBits(); const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); @@ -861,7 +1380,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return false; } else { // s32 and s64 are covered by tablegen. - if (Ty != p0) { + if (Ty != p0 && Ty != s8 && Ty != s16) { LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty << " constant, expected: " << s32 << ", " << s64 << ", or " << p0 << '\n'); @@ -876,25 +1395,27 @@ bool AArch64InstructionSelector::select(MachineInstr &I, } } + // We allow G_CONSTANT of types < 32b. const unsigned MovOpc = - DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm; - - I.setDesc(TII.get(MovOpc)); + DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm; if (isFP) { + // Either emit a FMOV, or emit a copy to emit a normal mov. const TargetRegisterClass &GPRRC = DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass; const TargetRegisterClass &FPRRC = DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass; - const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC); + // Can we use a FMOV instruction to represent the immediate? + if (emitFMovForFConstant(I, MRI)) + return true; + + // Nope. Emit a copy and use a normal mov instead. + const Register DefGPRReg = MRI.createVirtualRegister(&GPRRC); MachineOperand &RegOp = I.getOperand(0); RegOp.setReg(DefGPRReg); - - BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(), - TII.get(AArch64::COPY)) - .addDef(DefReg) - .addUse(DefGPRReg); + MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); + MIB.buildCopy({DefReg}, {DefGPRReg}); if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) { LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n"); @@ -913,6 +1434,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, I.getOperand(1).ChangeToImmediate(Val); } + I.setDesc(TII.get(MovOpc)); constrainSelectedInstRegOperands(I, TII, TRI, RBI); return true; } @@ -936,11 +1458,10 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } - unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); - BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(), - TII.get(AArch64::COPY)) - .addDef(I.getOperand(0).getReg()) - .addUse(DstReg, 0, AArch64::sub_32); + Register DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); + MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); + MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {}) + .addReg(DstReg, 0, AArch64::sub_32); RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR32RegClass, MRI); I.getOperand(0).setReg(DstReg); @@ -969,7 +1490,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } - unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); + Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); BuildMI(MBB, I.getIterator(), I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG)) .addDef(SrcReg) @@ -1026,8 +1547,12 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } + case TargetOpcode::G_ZEXTLOAD: case TargetOpcode::G_LOAD: case TargetOpcode::G_STORE: { + bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD; + MachineIRBuilder MIB(I); + LLT PtrTy = MRI.getType(I.getOperand(1).getReg()); if (PtrTy != LLT::pointer(0, 64)) { @@ -1043,7 +1568,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, } unsigned MemSizeInBits = MemOp.getSize() * 8; - const unsigned PtrReg = I.getOperand(1).getReg(); + const Register PtrReg = I.getOperand(1).getReg(); #ifndef NDEBUG const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI); // Sanity-check the pointer register. @@ -1053,7 +1578,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, "Load/Store pointer operand isn't a pointer"); #endif - const unsigned ValReg = I.getOperand(0).getReg(); + const Register ValReg = I.getOperand(0).getReg(); const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI); const unsigned NewOpc = @@ -1098,6 +1623,25 @@ bool AArch64InstructionSelector::select(MachineInstr &I, } } + if (IsZExtLoad) { + // The zextload from a smaller type to i32 should be handled by the importer. + if (MRI.getType(ValReg).getSizeInBits() != 64) + return false; + // If we have a ZEXTLOAD then change the load's type to be a narrower reg + //and zero_extend with SUBREG_TO_REG. + Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); + Register DstReg = I.getOperand(0).getReg(); + I.getOperand(0).setReg(LdReg); + + MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); + MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {}) + .addImm(0) + .addUse(LdReg) + .addImm(AArch64::sub_32); + constrainSelectedInstRegOperands(I, TII, TRI, RBI); + return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass, + MRI); + } return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } @@ -1107,7 +1651,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, if (unsupportedBinOp(I, RBI, MRI, TRI)) return false; - const unsigned DefReg = I.getOperand(0).getReg(); + const Register DefReg = I.getOperand(0).getReg(); const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); if (RB.getID() != AArch64::GPRRegBankID) { @@ -1134,10 +1678,17 @@ bool AArch64InstructionSelector::select(MachineInstr &I, case TargetOpcode::G_FMUL: case TargetOpcode::G_FDIV: - case TargetOpcode::G_OR: + case TargetOpcode::G_ASHR: + if (MRI.getType(I.getOperand(0).getReg()).isVector()) + return selectVectorASHR(I, MRI); + LLVM_FALLTHROUGH; case TargetOpcode::G_SHL: + if (Opcode == TargetOpcode::G_SHL && + MRI.getType(I.getOperand(0).getReg()).isVector()) + return selectVectorSHL(I, MRI); + LLVM_FALLTHROUGH; + case TargetOpcode::G_OR: case TargetOpcode::G_LSHR: - case TargetOpcode::G_ASHR: case TargetOpcode::G_GEP: { // Reject the various things we don't support yet. if (unsupportedBinOp(I, RBI, MRI, TRI)) @@ -1145,7 +1696,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, const unsigned OpSize = Ty.getSizeInBits(); - const unsigned DefReg = I.getOperand(0).getReg(); + const Register DefReg = I.getOperand(0).getReg(); const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize); @@ -1160,6 +1711,43 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } + case TargetOpcode::G_UADDO: { + // TODO: Support other types. + unsigned OpSize = Ty.getSizeInBits(); + if (OpSize != 32 && OpSize != 64) { + LLVM_DEBUG( + dbgs() + << "G_UADDO currently only supported for 32 and 64 b types.\n"); + return false; + } + + // TODO: Support vectors. + if (Ty.isVector()) { + LLVM_DEBUG(dbgs() << "G_UADDO currently only supported for scalars.\n"); + return false; + } + + // Add and set the set condition flag. + unsigned AddsOpc = OpSize == 32 ? AArch64::ADDSWrr : AArch64::ADDSXrr; + MachineIRBuilder MIRBuilder(I); + auto AddsMI = MIRBuilder.buildInstr( + AddsOpc, {I.getOperand(0).getReg()}, + {I.getOperand(2).getReg(), I.getOperand(3).getReg()}); + constrainSelectedInstRegOperands(*AddsMI, TII, TRI, RBI); + + // Now, put the overflow result in the register given by the first operand + // to the G_UADDO. CSINC increments the result when the predicate is false, + // so to get the increment when it's true, we need to use the inverse. In + // this case, we want to increment when carry is set. + auto CsetMI = MIRBuilder + .buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()}, + {Register(AArch64::WZR), Register(AArch64::WZR)}) + .addImm(getInvertedCondCode(AArch64CC::HS)); + constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI); + I.eraseFromParent(); + return true; + } + case TargetOpcode::G_PTR_MASK: { uint64_t Align = I.getOperand(2).getImm(); if (Align >= 64 || Align == 0) @@ -1176,8 +1764,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I, const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); const LLT SrcTy = MRI.getType(I.getOperand(1).getReg()); - const unsigned DstReg = I.getOperand(0).getReg(); - const unsigned SrcReg = I.getOperand(1).getReg(); + const Register DstReg = I.getOperand(0).getReg(); + const Register SrcReg = I.getOperand(1).getReg(); const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); @@ -1234,8 +1822,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I, } case TargetOpcode::G_ANYEXT: { - const unsigned DstReg = I.getOperand(0).getReg(); - const unsigned SrcReg = I.getOperand(1).getReg(); + const Register DstReg = I.getOperand(0).getReg(); + const Register SrcReg = I.getOperand(1).getReg(); const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI); if (RBDst.getID() != AArch64::GPRRegBankID) { @@ -1266,7 +1854,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, // At this point G_ANYEXT is just like a plain COPY, but we need // to explicitly form the 64-bit value if any. if (DstSize > 32) { - unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass); + Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass); BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG)) .addDef(ExtSrc) .addImm(0) @@ -1283,8 +1871,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I, const LLT DstTy = MRI.getType(I.getOperand(0).getReg()), SrcTy = MRI.getType(I.getOperand(1).getReg()); const bool isSigned = Opcode == TargetOpcode::G_SEXT; - const unsigned DefReg = I.getOperand(0).getReg(); - const unsigned SrcReg = I.getOperand(1).getReg(); + const Register DefReg = I.getOperand(0).getReg(); + const Register SrcReg = I.getOperand(1).getReg(); const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); if (RB.getID() != AArch64::GPRRegBankID) { @@ -1302,7 +1890,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return false; } - const unsigned SrcXReg = + const Register SrcXReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG)) .addDef(SrcXReg) @@ -1358,11 +1946,10 @@ bool AArch64InstructionSelector::select(MachineInstr &I, case TargetOpcode::G_BITCAST: // Imported SelectionDAG rules can handle every bitcast except those that // bitcast from a type to the same type. Ideally, these shouldn't occur - // but we might not run an optimizer that deletes them. - if (MRI.getType(I.getOperand(0).getReg()) == - MRI.getType(I.getOperand(1).getReg())) - return selectCopy(I, TII, MRI, TRI, RBI); - return false; + // but we might not run an optimizer that deletes them. The other exception + // is bitcasts involving pointer types, as SelectionDAG has no knowledge + // of them. + return selectCopy(I, TII, MRI, TRI, RBI); case TargetOpcode::G_SELECT: { if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) { @@ -1371,20 +1958,14 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return false; } - const unsigned CondReg = I.getOperand(1).getReg(); - const unsigned TReg = I.getOperand(2).getReg(); - const unsigned FReg = I.getOperand(3).getReg(); - - unsigned CSelOpc = 0; + const Register CondReg = I.getOperand(1).getReg(); + const Register TReg = I.getOperand(2).getReg(); + const Register FReg = I.getOperand(3).getReg(); - if (Ty == LLT::scalar(32)) { - CSelOpc = AArch64::CSELWr; - } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) { - CSelOpc = AArch64::CSELXr; - } else { - return false; - } + if (tryOptSelect(I)) + return true; + Register CSelOpc = selectSelectOpc(I, MRI, RBI); MachineInstr &TstMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri)) .addDef(AArch64::WZR) @@ -1404,48 +1985,21 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return true; } case TargetOpcode::G_ICMP: { + if (Ty.isVector()) + return selectVectorICmp(I, MRI); + if (Ty != LLT::scalar(32)) { LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty << ", expected: " << LLT::scalar(32) << '\n'); return false; } - unsigned CmpOpc = 0; - unsigned ZReg = 0; - - LLT CmpTy = MRI.getType(I.getOperand(2).getReg()); - if (CmpTy == LLT::scalar(32)) { - CmpOpc = AArch64::SUBSWrr; - ZReg = AArch64::WZR; - } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) { - CmpOpc = AArch64::SUBSXrr; - ZReg = AArch64::XZR; - } else { + MachineIRBuilder MIRBuilder(I); + if (!emitIntegerCompare(I.getOperand(2), I.getOperand(3), I.getOperand(1), + MIRBuilder)) return false; - } - - // CSINC increments the result by one when the condition code is false. - // Therefore, we have to invert the predicate to get an increment by 1 when - // the predicate is true. - const AArch64CC::CondCode invCC = - changeICMPPredToAArch64CC(CmpInst::getInversePredicate( - (CmpInst::Predicate)I.getOperand(1).getPredicate())); - - MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc)) - .addDef(ZReg) - .addUse(I.getOperand(2).getReg()) - .addUse(I.getOperand(3).getReg()); - - MachineInstr &CSetMI = - *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr)) - .addDef(I.getOperand(0).getReg()) - .addUse(AArch64::WZR) - .addUse(AArch64::WZR) - .addImm(invCC); - - constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI); - constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI); - + emitCSetForICMP(I.getOperand(0).getReg(), I.getOperand(1).getPredicate(), + MIRBuilder); I.eraseFromParent(); return true; } @@ -1457,15 +2011,9 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return false; } - unsigned CmpOpc = 0; - LLT CmpTy = MRI.getType(I.getOperand(2).getReg()); - if (CmpTy == LLT::scalar(32)) { - CmpOpc = AArch64::FCMPSrr; - } else if (CmpTy == LLT::scalar(64)) { - CmpOpc = AArch64::FCMPDrr; - } else { + unsigned CmpOpc = selectFCMPOpc(I, MRI); + if (!CmpOpc) return false; - } // FIXME: regbank @@ -1473,12 +2021,19 @@ bool AArch64InstructionSelector::select(MachineInstr &I, changeFCMPPredToAArch64CC( (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2); - MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc)) - .addUse(I.getOperand(2).getReg()) - .addUse(I.getOperand(3).getReg()); + // Partially build the compare. Decide if we need to add a use for the + // third operand based off whether or not we're comparing against 0.0. + auto CmpMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc)) + .addUse(I.getOperand(2).getReg()); + + // If we don't have an immediate compare, then we need to add a use of the + // register which wasn't used for the immediate. + // Note that the immediate will always be the last operand. + if (CmpOpc != AArch64::FCMPSri && CmpOpc != AArch64::FCMPDri) + CmpMI = CmpMI.addUse(I.getOperand(3).getReg()); - const unsigned DefReg = I.getOperand(0).getReg(); - unsigned Def1Reg = DefReg; + const Register DefReg = I.getOperand(0).getReg(); + Register Def1Reg = DefReg; if (CC2 != AArch64CC::AL) Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); @@ -1490,7 +2045,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, .addImm(getInvertedCondCode(CC1)); if (CC2 != AArch64CC::AL) { - unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); + Register Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); MachineInstr &CSet2MI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr)) .addDef(Def2Reg) @@ -1505,8 +2060,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I, constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI); constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI); } - - constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI); + constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI); constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI); I.eraseFromParent(); @@ -1515,19 +2069,14 @@ bool AArch64InstructionSelector::select(MachineInstr &I, case TargetOpcode::G_VASTART: return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI) : selectVaStartAAPCS(I, MF, MRI); + case TargetOpcode::G_INTRINSIC: + return selectIntrinsic(I, MRI); case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - if (!I.getOperand(0).isIntrinsicID()) - return false; - if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap) - return false; - BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::BRK)) - .addImm(1); - I.eraseFromParent(); - return true; + return selectIntrinsicWithSideEffects(I, MRI); case TargetOpcode::G_IMPLICIT_DEF: { I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF)); const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); - const unsigned DstReg = I.getOperand(0).getReg(); + const Register DstReg = I.getOperand(0).getReg(); const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(DstTy, DstRB, RBI); @@ -1552,44 +2101,374 @@ bool AArch64InstructionSelector::select(MachineInstr &I, return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI); } } + case TargetOpcode::G_INTRINSIC_TRUNC: + return selectIntrinsicTrunc(I, MRI); + case TargetOpcode::G_INTRINSIC_ROUND: + return selectIntrinsicRound(I, MRI); case TargetOpcode::G_BUILD_VECTOR: return selectBuildVector(I, MRI); case TargetOpcode::G_MERGE_VALUES: return selectMergeValues(I, MRI); + case TargetOpcode::G_UNMERGE_VALUES: + return selectUnmergeValues(I, MRI); + case TargetOpcode::G_SHUFFLE_VECTOR: + return selectShuffleVector(I, MRI); + case TargetOpcode::G_EXTRACT_VECTOR_ELT: + return selectExtractElt(I, MRI); + case TargetOpcode::G_INSERT_VECTOR_ELT: + return selectInsertElt(I, MRI); + case TargetOpcode::G_CONCAT_VECTORS: + return selectConcatVectors(I, MRI); + case TargetOpcode::G_JUMP_TABLE: + return selectJumpTable(I, MRI); } return false; } -bool AArch64InstructionSelector::emitScalarToVector( - unsigned &Dst, const LLT DstTy, const TargetRegisterClass *DstRC, - unsigned Scalar, MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, MachineRegisterInfo &MRI) const { - Dst = MRI.createVirtualRegister(DstRC); +bool AArch64InstructionSelector::selectBrJT(MachineInstr &I, + MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT"); + Register JTAddr = I.getOperand(0).getReg(); + unsigned JTI = I.getOperand(1).getIndex(); + Register Index = I.getOperand(2).getReg(); + MachineIRBuilder MIB(I); + + Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); + Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); + MIB.buildInstr(AArch64::JumpTableDest32, {TargetReg, ScratchReg}, + {JTAddr, Index}) + .addJumpTableIndex(JTI); + + // Build the indirect branch. + MIB.buildInstr(AArch64::BR, {}, {TargetReg}); + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::selectJumpTable( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table"); + assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!"); + + Register DstReg = I.getOperand(0).getReg(); + unsigned JTI = I.getOperand(1).getIndex(); + // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later. + MachineIRBuilder MIB(I); + auto MovMI = + MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {}) + .addJumpTableIndex(JTI, AArch64II::MO_PAGE) + .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF); + I.eraseFromParent(); + return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI); +} + +bool AArch64InstructionSelector::selectIntrinsicTrunc( + MachineInstr &I, MachineRegisterInfo &MRI) const { + const LLT SrcTy = MRI.getType(I.getOperand(0).getReg()); + + // Select the correct opcode. + unsigned Opc = 0; + if (!SrcTy.isVector()) { + switch (SrcTy.getSizeInBits()) { + default: + case 16: + Opc = AArch64::FRINTZHr; + break; + case 32: + Opc = AArch64::FRINTZSr; + break; + case 64: + Opc = AArch64::FRINTZDr; + break; + } + } else { + unsigned NumElts = SrcTy.getNumElements(); + switch (SrcTy.getElementType().getSizeInBits()) { + default: + break; + case 16: + if (NumElts == 4) + Opc = AArch64::FRINTZv4f16; + else if (NumElts == 8) + Opc = AArch64::FRINTZv8f16; + break; + case 32: + if (NumElts == 2) + Opc = AArch64::FRINTZv2f32; + else if (NumElts == 4) + Opc = AArch64::FRINTZv4f32; + break; + case 64: + if (NumElts == 2) + Opc = AArch64::FRINTZv2f64; + break; + } + } - unsigned UndefVec = MRI.createVirtualRegister(DstRC); - MachineInstr &UndefMI = *BuildMI(MBB, MBBI, MBBI->getDebugLoc(), - TII.get(TargetOpcode::IMPLICIT_DEF)) - .addDef(UndefVec); + if (!Opc) { + // Didn't get an opcode above, bail. + LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n"); + return false; + } + + // Legalization would have set us up perfectly for this; we just need to + // set the opcode and move on. + I.setDesc(TII.get(Opc)); + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); +} + +bool AArch64InstructionSelector::selectIntrinsicRound( + MachineInstr &I, MachineRegisterInfo &MRI) const { + const LLT SrcTy = MRI.getType(I.getOperand(0).getReg()); + + // Select the correct opcode. + unsigned Opc = 0; + if (!SrcTy.isVector()) { + switch (SrcTy.getSizeInBits()) { + default: + case 16: + Opc = AArch64::FRINTAHr; + break; + case 32: + Opc = AArch64::FRINTASr; + break; + case 64: + Opc = AArch64::FRINTADr; + break; + } + } else { + unsigned NumElts = SrcTy.getNumElements(); + switch (SrcTy.getElementType().getSizeInBits()) { + default: + break; + case 16: + if (NumElts == 4) + Opc = AArch64::FRINTAv4f16; + else if (NumElts == 8) + Opc = AArch64::FRINTAv8f16; + break; + case 32: + if (NumElts == 2) + Opc = AArch64::FRINTAv2f32; + else if (NumElts == 4) + Opc = AArch64::FRINTAv4f32; + break; + case 64: + if (NumElts == 2) + Opc = AArch64::FRINTAv2f64; + break; + } + } + + if (!Opc) { + // Didn't get an opcode above, bail. + LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n"); + return false; + } + + // Legalization would have set us up perfectly for this; we just need to + // set the opcode and move on. + I.setDesc(TII.get(Opc)); + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); +} + +bool AArch64InstructionSelector::selectVectorICmp( + MachineInstr &I, MachineRegisterInfo &MRI) const { + Register DstReg = I.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + Register SrcReg = I.getOperand(2).getReg(); + Register Src2Reg = I.getOperand(3).getReg(); + LLT SrcTy = MRI.getType(SrcReg); + + unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits(); + unsigned NumElts = DstTy.getNumElements(); + + // First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b + // Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16 + // Third index is cc opcode: + // 0 == eq + // 1 == ugt + // 2 == uge + // 3 == ult + // 4 == ule + // 5 == sgt + // 6 == sge + // 7 == slt + // 8 == sle + // ne is done by negating 'eq' result. + + // This table below assumes that for some comparisons the operands will be + // commuted. + // ult op == commute + ugt op + // ule op == commute + uge op + // slt op == commute + sgt op + // sle op == commute + sge op + unsigned PredIdx = 0; + bool SwapOperands = false; + CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate(); + switch (Pred) { + case CmpInst::ICMP_NE: + case CmpInst::ICMP_EQ: + PredIdx = 0; + break; + case CmpInst::ICMP_UGT: + PredIdx = 1; + break; + case CmpInst::ICMP_UGE: + PredIdx = 2; + break; + case CmpInst::ICMP_ULT: + PredIdx = 3; + SwapOperands = true; + break; + case CmpInst::ICMP_ULE: + PredIdx = 4; + SwapOperands = true; + break; + case CmpInst::ICMP_SGT: + PredIdx = 5; + break; + case CmpInst::ICMP_SGE: + PredIdx = 6; + break; + case CmpInst::ICMP_SLT: + PredIdx = 7; + SwapOperands = true; + break; + case CmpInst::ICMP_SLE: + PredIdx = 8; + SwapOperands = true; + break; + default: + llvm_unreachable("Unhandled icmp predicate"); + return false; + } + + // This table obviously should be tablegen'd when we have our GISel native + // tablegen selector. + + static const unsigned OpcTable[4][4][9] = { + { + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */}, + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */}, + {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8, + AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8, + AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8}, + {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8, + AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8, + AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8} + }, + { + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */}, + {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16, + AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16, + AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16}, + {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16, + AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16, + AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16}, + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */} + }, + { + {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32, + AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32, + AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32}, + {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32, + AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32, + AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32}, + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */}, + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */} + }, + { + {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64, + AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64, + AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64}, + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */}, + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */}, + {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, + 0 /* invalid */} + }, + }; + unsigned EltIdx = Log2_32(SrcEltSize / 8); + unsigned NumEltsIdx = Log2_32(NumElts / 2); + unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx]; + if (!Opc) { + LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode"); + return false; + } + + const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI); + const TargetRegisterClass *SrcRC = + getRegClassForTypeOnBank(SrcTy, VecRB, RBI, true); + if (!SrcRC) { + LLVM_DEBUG(dbgs() << "Could not determine source register class.\n"); + return false; + } + + unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0; + if (SrcTy.getSizeInBits() == 128) + NotOpc = NotOpc ? AArch64::NOTv16i8 : 0; + + if (SwapOperands) + std::swap(SrcReg, Src2Reg); + + MachineIRBuilder MIB(I); + auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg}); + constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI); + + // Invert if we had a 'ne' cc. + if (NotOpc) { + Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp}); + constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI); + } else { + MIB.buildCopy(DstReg, Cmp.getReg(0)); + } + RBI.constrainGenericRegister(DstReg, *SrcRC, MRI); + I.eraseFromParent(); + return true; +} + +MachineInstr *AArch64InstructionSelector::emitScalarToVector( + unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar, + MachineIRBuilder &MIRBuilder) const { + auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {}); auto BuildFn = [&](unsigned SubregIndex) { - MachineInstr &InsMI = *BuildMI(MBB, MBBI, MBBI->getDebugLoc(), - TII.get(TargetOpcode::INSERT_SUBREG)) - .addDef(Dst) - .addUse(UndefVec) - .addUse(Scalar) - .addImm(SubregIndex); - constrainSelectedInstRegOperands(UndefMI, TII, TRI, RBI); - return constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI); + auto Ins = + MIRBuilder + .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar}) + .addImm(SubregIndex); + constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI); + constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI); + return &*Ins; }; - switch (DstTy.getElementType().getSizeInBits()) { + switch (EltSize) { + case 16: + return BuildFn(AArch64::hsub); case 32: return BuildFn(AArch64::ssub); case 64: return BuildFn(AArch64::dsub); default: - return false; + return nullptr; } } @@ -1610,14 +2489,14 @@ bool AArch64InstructionSelector::selectMergeValues( return false; auto *DstRC = &AArch64::GPR64RegClass; - unsigned SubToRegDef = MRI.createVirtualRegister(DstRC); + Register SubToRegDef = MRI.createVirtualRegister(DstRC); MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::SUBREG_TO_REG)) .addDef(SubToRegDef) .addImm(0) .addUse(I.getOperand(1).getReg()) .addImm(AArch64::sub_32); - unsigned SubToRegDef2 = MRI.createVirtualRegister(DstRC); + Register SubToRegDef2 = MRI.createVirtualRegister(DstRC); // Need to anyext the second scalar before we can use bfm MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::SUBREG_TO_REG)) @@ -1639,122 +2518,1362 @@ bool AArch64InstructionSelector::selectMergeValues( return true; } -bool AArch64InstructionSelector::selectBuildVector( +static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg, + const unsigned EltSize) { + // Choose a lane copy opcode and subregister based off of the size of the + // vector's elements. + switch (EltSize) { + case 16: + CopyOpc = AArch64::CPYi16; + ExtractSubReg = AArch64::hsub; + break; + case 32: + CopyOpc = AArch64::CPYi32; + ExtractSubReg = AArch64::ssub; + break; + case 64: + CopyOpc = AArch64::CPYi64; + ExtractSubReg = AArch64::dsub; + break; + default: + // Unknown size, bail out. + LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n"); + return false; + } + return true; +} + +MachineInstr *AArch64InstructionSelector::emitExtractVectorElt( + Optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy, + Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const { + MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); + unsigned CopyOpc = 0; + unsigned ExtractSubReg = 0; + if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) { + LLVM_DEBUG( + dbgs() << "Couldn't determine lane copy opcode for instruction.\n"); + return nullptr; + } + + const TargetRegisterClass *DstRC = + getRegClassForTypeOnBank(ScalarTy, DstRB, RBI, true); + if (!DstRC) { + LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n"); + return nullptr; + } + + const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI); + const LLT &VecTy = MRI.getType(VecReg); + const TargetRegisterClass *VecRC = + getRegClassForTypeOnBank(VecTy, VecRB, RBI, true); + if (!VecRC) { + LLVM_DEBUG(dbgs() << "Could not determine source register class.\n"); + return nullptr; + } + + // The register that we're going to copy into. + Register InsertReg = VecReg; + if (!DstReg) + DstReg = MRI.createVirtualRegister(DstRC); + // If the lane index is 0, we just use a subregister COPY. + if (LaneIdx == 0) { + auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {}) + .addReg(VecReg, 0, ExtractSubReg); + RBI.constrainGenericRegister(*DstReg, *DstRC, MRI); + return &*Copy; + } + + // Lane copies require 128-bit wide registers. If we're dealing with an + // unpacked vector, then we need to move up to that width. Insert an implicit + // def and a subregister insert to get us there. + if (VecTy.getSizeInBits() != 128) { + MachineInstr *ScalarToVector = emitScalarToVector( + VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder); + if (!ScalarToVector) + return nullptr; + InsertReg = ScalarToVector->getOperand(0).getReg(); + } + + MachineInstr *LaneCopyMI = + MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx); + constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI); + + // Make sure that we actually constrain the initial copy. + RBI.constrainGenericRegister(*DstReg, *DstRC, MRI); + return LaneCopyMI; +} + +bool AArch64InstructionSelector::selectExtractElt( MachineInstr &I, MachineRegisterInfo &MRI) const { - assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR); - // Until we port more of the optimized selections, for now just use a vector - // insert sequence. - const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); - const LLT EltTy = MRI.getType(I.getOperand(1).getReg()); - unsigned EltSize = EltTy.getSizeInBits(); - if (EltSize < 32 || EltSize > 64) - return false; // Don't support all element types yet. - const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI); - unsigned Opc; - unsigned SubregIdx; + assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && + "unexpected opcode!"); + Register DstReg = I.getOperand(0).getReg(); + const LLT NarrowTy = MRI.getType(DstReg); + const Register SrcReg = I.getOperand(1).getReg(); + const LLT WideTy = MRI.getType(SrcReg); + (void)WideTy; + assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && + "source register size too small!"); + assert(NarrowTy.isScalar() && "cannot extract vector into vector!"); + + // Need the lane index to determine the correct copy opcode. + MachineOperand &LaneIdxOp = I.getOperand(2); + assert(LaneIdxOp.isReg() && "Lane index operand was not a register?"); + + if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) { + LLVM_DEBUG(dbgs() << "Cannot extract into GPR.\n"); + return false; + } + + // Find the index to extract from. + auto VRegAndVal = getConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI); + if (!VRegAndVal) + return false; + unsigned LaneIdx = VRegAndVal->Value; + + MachineIRBuilder MIRBuilder(I); + + const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); + MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg, + LaneIdx, MIRBuilder); + if (!Extract) + return false; + + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::selectSplitVectorUnmerge( + MachineInstr &I, MachineRegisterInfo &MRI) const { + unsigned NumElts = I.getNumOperands() - 1; + Register SrcReg = I.getOperand(NumElts).getReg(); + const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg()); + const LLT SrcTy = MRI.getType(SrcReg); + + assert(NarrowTy.isVector() && "Expected an unmerge into vectors"); + if (SrcTy.getSizeInBits() > 128) { + LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge"); + return false; + } + + MachineIRBuilder MIB(I); + + // We implement a split vector operation by treating the sub-vectors as + // scalars and extracting them. + const RegisterBank &DstRB = + *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI); + for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) { + Register Dst = I.getOperand(OpIdx).getReg(); + MachineInstr *Extract = + emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB); + if (!Extract) + return false; + } + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::selectUnmergeValues( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && + "unexpected opcode"); + + // TODO: Handle unmerging into GPRs and from scalars to scalars. + if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() != + AArch64::FPRRegBankID || + RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() != + AArch64::FPRRegBankID) { + LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar " + "currently unsupported.\n"); + return false; + } + + // The last operand is the vector source register, and every other operand is + // a register to unpack into. + unsigned NumElts = I.getNumOperands() - 1; + Register SrcReg = I.getOperand(NumElts).getReg(); + const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg()); + const LLT WideTy = MRI.getType(SrcReg); + (void)WideTy; + assert(WideTy.isVector() && "can only unmerge from vector types!"); + assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && + "source register size too small!"); + + if (!NarrowTy.isScalar()) + return selectSplitVectorUnmerge(I, MRI); + + MachineIRBuilder MIB(I); + + // Choose a lane copy opcode and subregister based off of the size of the + // vector's elements. + unsigned CopyOpc = 0; + unsigned ExtractSubReg = 0; + if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits())) + return false; + + // Set up for the lane copies. + MachineBasicBlock &MBB = *I.getParent(); + + // Stores the registers we'll be copying from. + SmallVector<Register, 4> InsertRegs; + + // We'll use the first register twice, so we only need NumElts-1 registers. + unsigned NumInsertRegs = NumElts - 1; + + // If our elements fit into exactly 128 bits, then we can copy from the source + // directly. Otherwise, we need to do a bit of setup with some subregister + // inserts. + if (NarrowTy.getSizeInBits() * NumElts == 128) { + InsertRegs = SmallVector<Register, 4>(NumInsertRegs, SrcReg); + } else { + // No. We have to perform subregister inserts. For each insert, create an + // implicit def and a subregister insert, and save the register we create. + for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) { + Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass); + MachineInstr &ImpDefMI = + *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF), + ImpDefReg); + + // Now, create the subregister insert from SrcReg. + Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass); + MachineInstr &InsMI = + *BuildMI(MBB, I, I.getDebugLoc(), + TII.get(TargetOpcode::INSERT_SUBREG), InsertReg) + .addUse(ImpDefReg) + .addUse(SrcReg) + .addImm(AArch64::dsub); + + constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI); + constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI); + + // Save the register so that we can copy from it after. + InsertRegs.push_back(InsertReg); + } + } + + // Now that we've created any necessary subregister inserts, we can + // create the copies. + // + // Perform the first copy separately as a subregister copy. + Register CopyTo = I.getOperand(0).getReg(); + auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {}) + .addReg(InsertRegs[0], 0, ExtractSubReg); + constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI); + + // Now, perform the remaining copies as vector lane copies. + unsigned LaneIdx = 1; + for (Register InsReg : InsertRegs) { + Register CopyTo = I.getOperand(LaneIdx).getReg(); + MachineInstr &CopyInst = + *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo) + .addUse(InsReg) + .addImm(LaneIdx); + constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI); + ++LaneIdx; + } + + // Separately constrain the first copy's destination. Because of the + // limitation in constrainOperandRegClass, we can't guarantee that this will + // actually be constrained. So, do it ourselves using the second operand. + const TargetRegisterClass *RC = + MRI.getRegClassOrNull(I.getOperand(1).getReg()); + if (!RC) { + LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n"); + return false; + } + + RBI.constrainGenericRegister(CopyTo, *RC, MRI); + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::selectConcatVectors( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && + "Unexpected opcode"); + Register Dst = I.getOperand(0).getReg(); + Register Op1 = I.getOperand(1).getReg(); + Register Op2 = I.getOperand(2).getReg(); + MachineIRBuilder MIRBuilder(I); + MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder); + if (!ConcatMI) + return false; + I.eraseFromParent(); + return true; +} + +void AArch64InstructionSelector::collectShuffleMaskIndices( + MachineInstr &I, MachineRegisterInfo &MRI, + SmallVectorImpl<Optional<int>> &Idxs) const { + MachineInstr *MaskDef = MRI.getVRegDef(I.getOperand(3).getReg()); + assert( + MaskDef->getOpcode() == TargetOpcode::G_BUILD_VECTOR && + "G_SHUFFLE_VECTOR should have a constant mask operand as G_BUILD_VECTOR"); + // Find the constant indices. + for (unsigned i = 1, e = MaskDef->getNumOperands(); i < e; ++i) { + // Look through copies. + MachineInstr *ScalarDef = + getDefIgnoringCopies(MaskDef->getOperand(i).getReg(), MRI); + assert(ScalarDef && "Could not find vreg def of shufflevec index op"); + if (ScalarDef->getOpcode() != TargetOpcode::G_CONSTANT) { + // This be an undef if not a constant. + assert(ScalarDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF); + Idxs.push_back(None); + } else { + Idxs.push_back(ScalarDef->getOperand(1).getCImm()->getSExtValue()); + } + } +} + +unsigned +AArch64InstructionSelector::emitConstantPoolEntry(Constant *CPVal, + MachineFunction &MF) const { + Type *CPTy = CPVal->getType(); + unsigned Align = MF.getDataLayout().getPrefTypeAlignment(CPTy); + if (Align == 0) + Align = MF.getDataLayout().getTypeAllocSize(CPTy); + + MachineConstantPool *MCP = MF.getConstantPool(); + return MCP->getConstantPoolIndex(CPVal, Align); +} + +MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool( + Constant *CPVal, MachineIRBuilder &MIRBuilder) const { + unsigned CPIdx = emitConstantPoolEntry(CPVal, MIRBuilder.getMF()); + + auto Adrp = + MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {}) + .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE); + + MachineInstr *LoadMI = nullptr; + switch (MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType())) { + case 16: + LoadMI = + &*MIRBuilder + .buildInstr(AArch64::LDRQui, {&AArch64::FPR128RegClass}, {Adrp}) + .addConstantPoolIndex(CPIdx, 0, + AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + break; + case 8: + LoadMI = &*MIRBuilder + .buildInstr(AArch64::LDRDui, {&AArch64::FPR64RegClass}, {Adrp}) + .addConstantPoolIndex( + CPIdx, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + break; + default: + LLVM_DEBUG(dbgs() << "Could not load from constant pool of type " + << *CPVal->getType()); + return nullptr; + } + constrainSelectedInstRegOperands(*Adrp, TII, TRI, RBI); + constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI); + return LoadMI; +} + +/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given +/// size and RB. +static std::pair<unsigned, unsigned> +getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) { + unsigned Opc, SubregIdx; if (RB.getID() == AArch64::GPRRegBankID) { if (EltSize == 32) { Opc = AArch64::INSvi32gpr; SubregIdx = AArch64::ssub; - } else { + } else if (EltSize == 64) { Opc = AArch64::INSvi64gpr; SubregIdx = AArch64::dsub; + } else { + llvm_unreachable("invalid elt size!"); } } else { - if (EltSize == 32) { + if (EltSize == 8) { + Opc = AArch64::INSvi8lane; + SubregIdx = AArch64::bsub; + } else if (EltSize == 16) { + Opc = AArch64::INSvi16lane; + SubregIdx = AArch64::hsub; + } else if (EltSize == 32) { Opc = AArch64::INSvi32lane; SubregIdx = AArch64::ssub; - } else { + } else if (EltSize == 64) { Opc = AArch64::INSvi64lane; SubregIdx = AArch64::dsub; + } else { + llvm_unreachable("invalid elt size!"); } } + return std::make_pair(Opc, SubregIdx); +} - if (EltSize * DstTy.getNumElements() != 128) - return false; // Don't handle unpacked vectors yet. +MachineInstr * +AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS, + MachineIRBuilder &MIRBuilder) const { + assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!"); + MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); + static const unsigned OpcTable[2][2]{{AArch64::ADDSXrr, AArch64::ADDSXri}, + {AArch64::ADDSWrr, AArch64::ADDSWri}}; + bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32); + auto ImmFns = selectArithImmed(RHS); + unsigned Opc = OpcTable[Is32Bit][ImmFns.hasValue()]; + Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR; - unsigned DstVec = 0; - const TargetRegisterClass *DstRC = getRegClassForTypeOnBank( - DstTy, RBI.getRegBank(AArch64::FPRRegBankID), RBI); - emitScalarToVector(DstVec, DstTy, DstRC, I.getOperand(1).getReg(), - *I.getParent(), I.getIterator(), MRI); - for (unsigned i = 2, e = DstTy.getSizeInBits() / EltSize + 1; i < e; ++i) { - unsigned InsDef; - // For the last insert re-use the dst reg of the G_BUILD_VECTOR. - if (i + 1 < e) - InsDef = MRI.createVirtualRegister(DstRC); - else - InsDef = I.getOperand(0).getReg(); - unsigned LaneIdx = i - 1; - if (RB.getID() == AArch64::FPRRegBankID) { - unsigned ImpDef = MRI.createVirtualRegister(DstRC); - MachineInstr &ImpDefMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(), - TII.get(TargetOpcode::IMPLICIT_DEF)) - .addDef(ImpDef); - unsigned InsSubDef = MRI.createVirtualRegister(DstRC); - MachineInstr &InsSubMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(), - TII.get(TargetOpcode::INSERT_SUBREG)) - .addDef(InsSubDef) - .addUse(ImpDef) - .addUse(I.getOperand(i).getReg()) - .addImm(SubregIdx); - MachineInstr &InsEltMI = - *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opc)) - .addDef(InsDef) - .addUse(DstVec) - .addImm(LaneIdx) - .addUse(InsSubDef) - .addImm(0); - constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI); - constrainSelectedInstRegOperands(InsSubMI, TII, TRI, RBI); - constrainSelectedInstRegOperands(InsEltMI, TII, TRI, RBI); - DstVec = InsDef; - } else { - MachineInstr &InsMI = - *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opc)) - .addDef(InsDef) - .addUse(DstVec) - .addImm(LaneIdx) - .addUse(I.getOperand(i).getReg()); - constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI); - DstVec = InsDef; + auto CmpMI = MIRBuilder.buildInstr(Opc, {ZReg}, {LHS.getReg()}); + + // If we matched a valid constant immediate, add those operands. + if (ImmFns) { + for (auto &RenderFn : *ImmFns) + RenderFn(CmpMI); + } else { + CmpMI.addUse(RHS.getReg()); + } + + constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI); + return &*CmpMI; +} + +MachineInstr * +AArch64InstructionSelector::emitTST(const Register &LHS, const Register &RHS, + MachineIRBuilder &MIRBuilder) const { + MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); + unsigned RegSize = MRI.getType(LHS).getSizeInBits(); + bool Is32Bit = (RegSize == 32); + static const unsigned OpcTable[2][2]{{AArch64::ANDSXrr, AArch64::ANDSXri}, + {AArch64::ANDSWrr, AArch64::ANDSWri}}; + Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR; + + // We might be able to fold in an immediate into the TST. We need to make sure + // it's a logical immediate though, since ANDS requires that. + auto ValAndVReg = getConstantVRegValWithLookThrough(RHS, MRI); + bool IsImmForm = ValAndVReg.hasValue() && + AArch64_AM::isLogicalImmediate(ValAndVReg->Value, RegSize); + unsigned Opc = OpcTable[Is32Bit][IsImmForm]; + auto TstMI = MIRBuilder.buildInstr(Opc, {ZReg}, {LHS}); + + if (IsImmForm) + TstMI.addImm( + AArch64_AM::encodeLogicalImmediate(ValAndVReg->Value, RegSize)); + else + TstMI.addUse(RHS); + + constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI); + return &*TstMI; +} + +MachineInstr *AArch64InstructionSelector::emitIntegerCompare( + MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate, + MachineIRBuilder &MIRBuilder) const { + assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!"); + MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); + + // Fold the compare if possible. + MachineInstr *FoldCmp = + tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder); + if (FoldCmp) + return FoldCmp; + + // Can't fold into a CMN. Just emit a normal compare. + unsigned CmpOpc = 0; + Register ZReg; + + LLT CmpTy = MRI.getType(LHS.getReg()); + assert((CmpTy.isScalar() || CmpTy.isPointer()) && + "Expected scalar or pointer"); + if (CmpTy == LLT::scalar(32)) { + CmpOpc = AArch64::SUBSWrr; + ZReg = AArch64::WZR; + } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) { + CmpOpc = AArch64::SUBSXrr; + ZReg = AArch64::XZR; + } else { + return nullptr; + } + + // Try to match immediate forms. + auto ImmFns = selectArithImmed(RHS); + if (ImmFns) + CmpOpc = CmpOpc == AArch64::SUBSWrr ? AArch64::SUBSWri : AArch64::SUBSXri; + + auto CmpMI = MIRBuilder.buildInstr(CmpOpc).addDef(ZReg).addUse(LHS.getReg()); + // If we matched a valid constant immediate, add those operands. + if (ImmFns) { + for (auto &RenderFn : *ImmFns) + RenderFn(CmpMI); + } else { + CmpMI.addUse(RHS.getReg()); + } + + // Make sure that we can constrain the compare that we emitted. + constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI); + return &*CmpMI; +} + +MachineInstr *AArch64InstructionSelector::emitVectorConcat( + Optional<Register> Dst, Register Op1, Register Op2, + MachineIRBuilder &MIRBuilder) const { + // We implement a vector concat by: + // 1. Use scalar_to_vector to insert the lower vector into the larger dest + // 2. Insert the upper vector into the destination's upper element + // TODO: some of this code is common with G_BUILD_VECTOR handling. + MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); + + const LLT Op1Ty = MRI.getType(Op1); + const LLT Op2Ty = MRI.getType(Op2); + + if (Op1Ty != Op2Ty) { + LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys"); + return nullptr; + } + assert(Op1Ty.isVector() && "Expected a vector for vector concat"); + + if (Op1Ty.getSizeInBits() >= 128) { + LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors"); + return nullptr; + } + + // At the moment we just support 64 bit vector concats. + if (Op1Ty.getSizeInBits() != 64) { + LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors"); + return nullptr; + } + + const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits()); + const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI); + const TargetRegisterClass *DstRC = + getMinClassForRegBank(FPRBank, Op1Ty.getSizeInBits() * 2); + + MachineInstr *WidenedOp1 = + emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder); + MachineInstr *WidenedOp2 = + emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder); + if (!WidenedOp1 || !WidenedOp2) { + LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value"); + return nullptr; + } + + // Now do the insert of the upper element. + unsigned InsertOpc, InsSubRegIdx; + std::tie(InsertOpc, InsSubRegIdx) = + getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits()); + + if (!Dst) + Dst = MRI.createVirtualRegister(DstRC); + auto InsElt = + MIRBuilder + .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()}) + .addImm(1) /* Lane index */ + .addUse(WidenedOp2->getOperand(0).getReg()) + .addImm(0); + constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI); + return &*InsElt; +} + +MachineInstr *AArch64InstructionSelector::emitFMovForFConstant( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && + "Expected a G_FCONSTANT!"); + MachineOperand &ImmOp = I.getOperand(1); + unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits(); + + // Only handle 32 and 64 bit defs for now. + if (DefSize != 32 && DefSize != 64) + return nullptr; + + // Don't handle null values using FMOV. + if (ImmOp.getFPImm()->isNullValue()) + return nullptr; + + // Get the immediate representation for the FMOV. + const APFloat &ImmValAPF = ImmOp.getFPImm()->getValueAPF(); + int Imm = DefSize == 32 ? AArch64_AM::getFP32Imm(ImmValAPF) + : AArch64_AM::getFP64Imm(ImmValAPF); + + // If this is -1, it means the immediate can't be represented as the requested + // floating point value. Bail. + if (Imm == -1) + return nullptr; + + // Update MI to represent the new FMOV instruction, constrain it, and return. + ImmOp.ChangeToImmediate(Imm); + unsigned MovOpc = DefSize == 32 ? AArch64::FMOVSi : AArch64::FMOVDi; + I.setDesc(TII.get(MovOpc)); + constrainSelectedInstRegOperands(I, TII, TRI, RBI); + return &I; +} + +MachineInstr * +AArch64InstructionSelector::emitCSetForICMP(Register DefReg, unsigned Pred, + MachineIRBuilder &MIRBuilder) const { + // CSINC increments the result when the predicate is false. Invert it. + const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC( + CmpInst::getInversePredicate((CmpInst::Predicate)Pred)); + auto I = + MIRBuilder + .buildInstr(AArch64::CSINCWr, {DefReg}, {Register(AArch64::WZR), Register(AArch64::WZR)}) + .addImm(InvCC); + constrainSelectedInstRegOperands(*I, TII, TRI, RBI); + return &*I; +} + +bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) const { + MachineIRBuilder MIB(I); + MachineRegisterInfo &MRI = *MIB.getMRI(); + const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + + // We want to recognize this pattern: + // + // $z = G_FCMP pred, $x, $y + // ... + // $w = G_SELECT $z, $a, $b + // + // Where the value of $z is *only* ever used by the G_SELECT (possibly with + // some copies/truncs in between.) + // + // If we see this, then we can emit something like this: + // + // fcmp $x, $y + // fcsel $w, $a, $b, pred + // + // Rather than emitting both of the rather long sequences in the standard + // G_FCMP/G_SELECT select methods. + + // First, check if the condition is defined by a compare. + MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg()); + while (CondDef) { + // We can only fold if all of the defs have one use. + if (!MRI.hasOneUse(CondDef->getOperand(0).getReg())) + return false; + + // We can skip over G_TRUNC since the condition is 1-bit. + // Truncating/extending can have no impact on the value. + unsigned Opc = CondDef->getOpcode(); + if (Opc != TargetOpcode::COPY && Opc != TargetOpcode::G_TRUNC) + break; + + // Can't see past copies from physregs. + if (Opc == TargetOpcode::COPY && + TargetRegisterInfo::isPhysicalRegister(CondDef->getOperand(1).getReg())) + return false; + + CondDef = MRI.getVRegDef(CondDef->getOperand(1).getReg()); + } + + // Is the condition defined by a compare? + if (!CondDef) + return false; + + unsigned CondOpc = CondDef->getOpcode(); + if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) + return false; + + AArch64CC::CondCode CondCode; + if (CondOpc == TargetOpcode::G_ICMP) { + CondCode = changeICMPPredToAArch64CC( + (CmpInst::Predicate)CondDef->getOperand(1).getPredicate()); + if (!emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3), + CondDef->getOperand(1), MIB)) { + LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n"); + return false; } + } else { + // Get the condition code for the select. + AArch64CC::CondCode CondCode2; + changeFCMPPredToAArch64CC( + (CmpInst::Predicate)CondDef->getOperand(1).getPredicate(), CondCode, + CondCode2); + + // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two + // instructions to emit the comparison. + // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be + // unnecessary. + if (CondCode2 != AArch64CC::AL) + return false; + + // Make sure we'll be able to select the compare. + unsigned CmpOpc = selectFCMPOpc(*CondDef, MRI); + if (!CmpOpc) + return false; + + // Emit a new compare. + auto Cmp = MIB.buildInstr(CmpOpc, {}, {CondDef->getOperand(2).getReg()}); + if (CmpOpc != AArch64::FCMPSri && CmpOpc != AArch64::FCMPDri) + Cmp.addUse(CondDef->getOperand(3).getReg()); + constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI); } + + // Emit the select. + unsigned CSelOpc = selectSelectOpc(I, MRI, RBI); + auto CSel = + MIB.buildInstr(CSelOpc, {I.getOperand(0).getReg()}, + {I.getOperand(2).getReg(), I.getOperand(3).getReg()}) + .addImm(CondCode); + constrainSelectedInstRegOperands(*CSel, TII, TRI, RBI); I.eraseFromParent(); return true; } -/// SelectArithImmed - Select an immediate value that can be represented as -/// a 12-bit value shifted left by either 0 or 12. If so, return true with -/// Val set to the 12-bit value and Shift set to the shifter operand. -InstructionSelector::ComplexRendererFns -AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const { - MachineInstr &MI = *Root.getParent(); - MachineBasicBlock &MBB = *MI.getParent(); +MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare( + MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate, + MachineIRBuilder &MIRBuilder) const { + assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && + "Unexpected MachineOperand"); + MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); + // We want to find this sort of thing: + // x = G_SUB 0, y + // G_ICMP z, x + // + // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead. + // e.g: + // + // cmn z, y + + // Helper lambda to detect the subtract followed by the compare. + // Takes in the def of the LHS or RHS, and checks if it's a subtract from 0. + auto IsCMN = [&](MachineInstr *DefMI, const AArch64CC::CondCode &CC) { + if (!DefMI || DefMI->getOpcode() != TargetOpcode::G_SUB) + return false; + + // Need to make sure NZCV is the same at the end of the transformation. + if (CC != AArch64CC::EQ && CC != AArch64CC::NE) + return false; + + // We want to match against SUBs. + if (DefMI->getOpcode() != TargetOpcode::G_SUB) + return false; + + // Make sure that we're getting + // x = G_SUB 0, y + auto ValAndVReg = + getConstantVRegValWithLookThrough(DefMI->getOperand(1).getReg(), MRI); + if (!ValAndVReg || ValAndVReg->Value != 0) + return false; + + // This can safely be represented as a CMN. + return true; + }; + + // Check if the RHS or LHS of the G_ICMP is defined by a SUB + MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI); + MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI); + CmpInst::Predicate P = (CmpInst::Predicate)Predicate.getPredicate(); + const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(P); + + // Given this: + // + // x = G_SUB 0, y + // G_ICMP x, z + // + // Produce this: + // + // cmn y, z + if (IsCMN(LHSDef, CC)) + return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder); + + // Same idea here, but with the RHS of the compare instead: + // + // Given this: + // + // x = G_SUB 0, y + // G_ICMP z, x + // + // Produce this: + // + // cmn z, y + if (IsCMN(RHSDef, CC)) + return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder); + + // Given this: + // + // z = G_AND x, y + // G_ICMP z, 0 + // + // Produce this if the compare is signed: + // + // tst x, y + if (!isUnsignedICMPPred(P) && LHSDef && + LHSDef->getOpcode() == TargetOpcode::G_AND) { + // Make sure that the RHS is 0. + auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI); + if (!ValAndVReg || ValAndVReg->Value != 0) + return nullptr; + + return emitTST(LHSDef->getOperand(1).getReg(), + LHSDef->getOperand(2).getReg(), MIRBuilder); + } + + return nullptr; +} + +bool AArch64InstructionSelector::tryOptVectorDup(MachineInstr &I) const { + // Try to match a vector splat operation into a dup instruction. + // We're looking for this pattern: + // %scalar:gpr(s64) = COPY $x0 + // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF + // %cst0:gpr(s32) = G_CONSTANT i32 0 + // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32) + // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32) + // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef, + // %zerovec(<2 x s32>) + // + // ...into: + // %splat = DUP %scalar + // We use the regbank of the scalar to determine which kind of dup to use. + MachineIRBuilder MIB(I); + MachineRegisterInfo &MRI = *MIB.getMRI(); + const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + using namespace TargetOpcode; + using namespace MIPatternMatch; + + // Begin matching the insert. + auto *InsMI = + getOpcodeDef(G_INSERT_VECTOR_ELT, I.getOperand(1).getReg(), MRI); + if (!InsMI) + return false; + // Match the undef vector operand. + auto *UndefMI = + getOpcodeDef(G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(), MRI); + if (!UndefMI) + return false; + // Match the scalar being splatted. + Register ScalarReg = InsMI->getOperand(2).getReg(); + const RegisterBank *ScalarRB = RBI.getRegBank(ScalarReg, MRI, TRI); + // Match the index constant 0. + int64_t Index = 0; + if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ICst(Index)) || Index) + return false; + + // The shuffle's second operand doesn't matter if the mask is all zero. + auto *ZeroVec = getOpcodeDef(G_BUILD_VECTOR, I.getOperand(3).getReg(), MRI); + if (!ZeroVec) + return false; + int64_t Zero = 0; + if (!mi_match(ZeroVec->getOperand(1).getReg(), MRI, m_ICst(Zero)) || Zero) + return false; + for (unsigned i = 1, e = ZeroVec->getNumOperands() - 1; i < e; ++i) { + if (ZeroVec->getOperand(i).getReg() != ZeroVec->getOperand(1).getReg()) + return false; // This wasn't an all zeros vector. + } + + // We're done, now find out what kind of splat we need. + LLT VecTy = MRI.getType(I.getOperand(0).getReg()); + LLT EltTy = VecTy.getElementType(); + if (VecTy.getSizeInBits() != 128 || EltTy.getSizeInBits() < 32) { + LLVM_DEBUG(dbgs() << "Could not optimize splat pattern < 128b yet"); + return false; + } + bool IsFP = ScalarRB->getID() == AArch64::FPRRegBankID; + static const unsigned OpcTable[2][2] = { + {AArch64::DUPv4i32gpr, AArch64::DUPv2i64gpr}, + {AArch64::DUPv4i32lane, AArch64::DUPv2i64lane}}; + unsigned Opc = OpcTable[IsFP][EltTy.getSizeInBits() == 64]; + + // For FP splats, we need to widen the scalar reg via undef too. + if (IsFP) { + MachineInstr *Widen = emitScalarToVector( + EltTy.getSizeInBits(), &AArch64::FPR128RegClass, ScalarReg, MIB); + if (!Widen) + return false; + ScalarReg = Widen->getOperand(0).getReg(); + } + auto Dup = MIB.buildInstr(Opc, {I.getOperand(0).getReg()}, {ScalarReg}); + if (IsFP) + Dup.addImm(0); + constrainSelectedInstRegOperands(*Dup, TII, TRI, RBI); + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::tryOptVectorShuffle(MachineInstr &I) const { + if (TM.getOptLevel() == CodeGenOpt::None) + return false; + if (tryOptVectorDup(I)) + return true; + return false; +} + +bool AArch64InstructionSelector::selectShuffleVector( + MachineInstr &I, MachineRegisterInfo &MRI) const { + if (tryOptVectorShuffle(I)) + return true; + const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); + Register Src1Reg = I.getOperand(1).getReg(); + const LLT Src1Ty = MRI.getType(Src1Reg); + Register Src2Reg = I.getOperand(2).getReg(); + const LLT Src2Ty = MRI.getType(Src2Reg); + + MachineBasicBlock &MBB = *I.getParent(); MachineFunction &MF = *MBB.getParent(); - MachineRegisterInfo &MRI = MF.getRegInfo(); + LLVMContext &Ctx = MF.getFunction().getContext(); - // This function is called from the addsub_shifted_imm ComplexPattern, - // which lists [imm] as the list of opcode it's interested in, however - // we still need to check whether the operand is actually an immediate - // here because the ComplexPattern opcode list is only used in - // root-level opcode matching. + // G_SHUFFLE_VECTOR doesn't really have a strictly enforced constant mask + // operand, it comes in as a normal vector value which we have to analyze to + // find the mask indices. If the mask element is undef, then + // collectShuffleMaskIndices() will add a None entry for that index into + // the list. + SmallVector<Optional<int>, 8> Mask; + collectShuffleMaskIndices(I, MRI, Mask); + assert(!Mask.empty() && "Expected to find mask indices"); + + // G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if + // it's originated from a <1 x T> type. Those should have been lowered into + // G_BUILD_VECTOR earlier. + if (!Src1Ty.isVector() || !Src2Ty.isVector()) { + LLVM_DEBUG(dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n"); + return false; + } + + unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8; + + SmallVector<Constant *, 64> CstIdxs; + for (auto &MaybeVal : Mask) { + // For now, any undef indexes we'll just assume to be 0. This should be + // optimized in future, e.g. to select DUP etc. + int Val = MaybeVal.hasValue() ? *MaybeVal : 0; + for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) { + unsigned Offset = Byte + Val * BytesPerElt; + CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset)); + } + } + + MachineIRBuilder MIRBuilder(I); + + // Use a constant pool to load the index vector for TBL. + Constant *CPVal = ConstantVector::get(CstIdxs); + MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder); + if (!IndexLoad) { + LLVM_DEBUG(dbgs() << "Could not load from a constant pool"); + return false; + } + + if (DstTy.getSizeInBits() != 128) { + assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty"); + // This case can be done with TBL1. + MachineInstr *Concat = emitVectorConcat(None, Src1Reg, Src2Reg, MIRBuilder); + if (!Concat) { + LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1"); + return false; + } + + // The constant pool load will be 64 bits, so need to convert to FPR128 reg. + IndexLoad = + emitScalarToVector(64, &AArch64::FPR128RegClass, + IndexLoad->getOperand(0).getReg(), MIRBuilder); + + auto TBL1 = MIRBuilder.buildInstr( + AArch64::TBLv16i8One, {&AArch64::FPR128RegClass}, + {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()}); + constrainSelectedInstRegOperands(*TBL1, TII, TRI, RBI); + + auto Copy = + MIRBuilder + .buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {}) + .addReg(TBL1.getReg(0), 0, AArch64::dsub); + RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI); + I.eraseFromParent(); + return true; + } + + // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive + // Q registers for regalloc. + auto RegSeq = MIRBuilder + .buildInstr(TargetOpcode::REG_SEQUENCE, + {&AArch64::QQRegClass}, {Src1Reg}) + .addImm(AArch64::qsub0) + .addUse(Src2Reg) + .addImm(AArch64::qsub1); + + auto TBL2 = + MIRBuilder.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0).getReg()}, + {RegSeq, IndexLoad->getOperand(0).getReg()}); + constrainSelectedInstRegOperands(*RegSeq, TII, TRI, RBI); + constrainSelectedInstRegOperands(*TBL2, TII, TRI, RBI); + I.eraseFromParent(); + return true; +} + +MachineInstr *AArch64InstructionSelector::emitLaneInsert( + Optional<Register> DstReg, Register SrcReg, Register EltReg, + unsigned LaneIdx, const RegisterBank &RB, + MachineIRBuilder &MIRBuilder) const { + MachineInstr *InsElt = nullptr; + const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass; + MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); + + // Create a register to define with the insert if one wasn't passed in. + if (!DstReg) + DstReg = MRI.createVirtualRegister(DstRC); + + unsigned EltSize = MRI.getType(EltReg).getSizeInBits(); + unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first; + + if (RB.getID() == AArch64::FPRRegBankID) { + auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder); + InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg}) + .addImm(LaneIdx) + .addUse(InsSub->getOperand(0).getReg()) + .addImm(0); + } else { + InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg}) + .addImm(LaneIdx) + .addUse(EltReg); + } + + constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI); + return InsElt; +} + +bool AArch64InstructionSelector::selectInsertElt( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT); + + // Get information on the destination. + Register DstReg = I.getOperand(0).getReg(); + const LLT DstTy = MRI.getType(DstReg); + unsigned VecSize = DstTy.getSizeInBits(); + + // Get information on the element we want to insert into the destination. + Register EltReg = I.getOperand(2).getReg(); + const LLT EltTy = MRI.getType(EltReg); + unsigned EltSize = EltTy.getSizeInBits(); + if (EltSize < 16 || EltSize > 64) + return false; // Don't support all element types yet. + + // Find the definition of the index. Bail out if it's not defined by a + // G_CONSTANT. + Register IdxReg = I.getOperand(3).getReg(); + auto VRegAndVal = getConstantVRegValWithLookThrough(IdxReg, MRI); + if (!VRegAndVal) + return false; + unsigned LaneIdx = VRegAndVal->Value; + + // Perform the lane insert. + Register SrcReg = I.getOperand(1).getReg(); + const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI); + MachineIRBuilder MIRBuilder(I); + + if (VecSize < 128) { + // If the vector we're inserting into is smaller than 128 bits, widen it + // to 128 to do the insert. + MachineInstr *ScalarToVec = emitScalarToVector( + VecSize, &AArch64::FPR128RegClass, SrcReg, MIRBuilder); + if (!ScalarToVec) + return false; + SrcReg = ScalarToVec->getOperand(0).getReg(); + } + + // Create an insert into a new FPR128 register. + // Note that if our vector is already 128 bits, we end up emitting an extra + // register. + MachineInstr *InsMI = + emitLaneInsert(None, SrcReg, EltReg, LaneIdx, EltRB, MIRBuilder); + + if (VecSize < 128) { + // If we had to widen to perform the insert, then we have to demote back to + // the original size to get the result we want. + Register DemoteVec = InsMI->getOperand(0).getReg(); + const TargetRegisterClass *RC = + getMinClassForRegBank(*RBI.getRegBank(DemoteVec, MRI, TRI), VecSize); + if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) { + LLVM_DEBUG(dbgs() << "Unsupported register class!\n"); + return false; + } + unsigned SubReg = 0; + if (!getSubRegForClass(RC, TRI, SubReg)) + return false; + if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) { + LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << VecSize + << "\n"); + return false; + } + MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {}) + .addReg(DemoteVec, 0, SubReg); + RBI.constrainGenericRegister(DstReg, *RC, MRI); + } else { + // No widening needed. + InsMI->getOperand(0).setReg(DstReg); + constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI); + } + + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::selectBuildVector( + MachineInstr &I, MachineRegisterInfo &MRI) const { + assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR); + // Until we port more of the optimized selections, for now just use a vector + // insert sequence. + const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); + const LLT EltTy = MRI.getType(I.getOperand(1).getReg()); + unsigned EltSize = EltTy.getSizeInBits(); + if (EltSize < 16 || EltSize > 64) + return false; // Don't support all element types yet. + const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI); + MachineIRBuilder MIRBuilder(I); + + const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass; + MachineInstr *ScalarToVec = + emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC, + I.getOperand(1).getReg(), MIRBuilder); + if (!ScalarToVec) + return false; + + Register DstVec = ScalarToVec->getOperand(0).getReg(); + unsigned DstSize = DstTy.getSizeInBits(); + + // Keep track of the last MI we inserted. Later on, we might be able to save + // a copy using it. + MachineInstr *PrevMI = nullptr; + for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) { + // Note that if we don't do a subregister copy, we can end up making an + // extra register. + PrevMI = &*emitLaneInsert(None, DstVec, I.getOperand(i).getReg(), i - 1, RB, + MIRBuilder); + DstVec = PrevMI->getOperand(0).getReg(); + } + + // If DstTy's size in bits is less than 128, then emit a subregister copy + // from DstVec to the last register we've defined. + if (DstSize < 128) { + // Force this to be FPR using the destination vector. + const TargetRegisterClass *RC = + getMinClassForRegBank(*RBI.getRegBank(DstVec, MRI, TRI), DstSize); + if (!RC) + return false; + if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) { + LLVM_DEBUG(dbgs() << "Unsupported register class!\n"); + return false; + } + + unsigned SubReg = 0; + if (!getSubRegForClass(RC, TRI, SubReg)) + return false; + if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) { + LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSize + << "\n"); + return false; + } + + Register Reg = MRI.createVirtualRegister(RC); + Register DstReg = I.getOperand(0).getReg(); + + MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {}) + .addReg(DstVec, 0, SubReg); + MachineOperand &RegOp = I.getOperand(1); + RegOp.setReg(Reg); + RBI.constrainGenericRegister(DstReg, *RC, MRI); + } else { + // We don't need a subregister copy. Save a copy by re-using the + // destination register on the final insert. + assert(PrevMI && "PrevMI was null?"); + PrevMI->getOperand(0).setReg(I.getOperand(0).getReg()); + constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI); + } + + I.eraseFromParent(); + return true; +} + +/// Helper function to find an intrinsic ID on an a MachineInstr. Returns the +/// ID if it exists, and 0 otherwise. +static unsigned findIntrinsicID(MachineInstr &I) { + auto IntrinOp = find_if(I.operands(), [&](const MachineOperand &Op) { + return Op.isIntrinsicID(); + }); + if (IntrinOp == I.operands_end()) + return 0; + return IntrinOp->getIntrinsicID(); +} + +/// Helper function to emit the correct opcode for a llvm.aarch64.stlxr +/// intrinsic. +static unsigned getStlxrOpcode(unsigned NumBytesToStore) { + switch (NumBytesToStore) { + // TODO: 1, 2, and 4 byte stores. + case 8: + return AArch64::STLXRX; + default: + LLVM_DEBUG(dbgs() << "Unexpected number of bytes to store! (" + << NumBytesToStore << ")\n"); + break; + } + return 0; +} + +bool AArch64InstructionSelector::selectIntrinsicWithSideEffects( + MachineInstr &I, MachineRegisterInfo &MRI) const { + // Find the intrinsic ID. + unsigned IntrinID = findIntrinsicID(I); + if (!IntrinID) + return false; + MachineIRBuilder MIRBuilder(I); + + // Select the instruction. + switch (IntrinID) { + default: + return false; + case Intrinsic::trap: + MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(1); + break; + case Intrinsic::debugtrap: + if (!STI.isTargetWindows()) + return false; + MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000); + break; + case Intrinsic::aarch64_stlxr: + Register StatReg = I.getOperand(0).getReg(); + assert(RBI.getSizeInBits(StatReg, MRI, TRI) == 32 && + "Status register must be 32 bits!"); + Register SrcReg = I.getOperand(2).getReg(); + + if (RBI.getSizeInBits(SrcReg, MRI, TRI) != 64) { + LLVM_DEBUG(dbgs() << "Only support 64-bit sources right now.\n"); + return false; + } + + Register PtrReg = I.getOperand(3).getReg(); + assert(MRI.getType(PtrReg).isPointer() && "Expected pointer operand"); + + // Expect only one memory operand. + if (!I.hasOneMemOperand()) + return false; + + const MachineMemOperand *MemOp = *I.memoperands_begin(); + unsigned NumBytesToStore = MemOp->getSize(); + unsigned Opc = getStlxrOpcode(NumBytesToStore); + if (!Opc) + return false; + + auto StoreMI = MIRBuilder.buildInstr(Opc, {StatReg}, {SrcReg, PtrReg}); + constrainSelectedInstRegOperands(*StoreMI, TII, TRI, RBI); + } + + I.eraseFromParent(); + return true; +} + +bool AArch64InstructionSelector::selectIntrinsic( + MachineInstr &I, MachineRegisterInfo &MRI) const { + unsigned IntrinID = findIntrinsicID(I); + if (!IntrinID) + return false; + MachineIRBuilder MIRBuilder(I); + + switch (IntrinID) { + default: + break; + case Intrinsic::aarch64_crypto_sha1h: + Register DstReg = I.getOperand(0).getReg(); + Register SrcReg = I.getOperand(2).getReg(); + + // FIXME: Should this be an assert? + if (MRI.getType(DstReg).getSizeInBits() != 32 || + MRI.getType(SrcReg).getSizeInBits() != 32) + return false; + + // The operation has to happen on FPRs. Set up some new FPR registers for + // the source and destination if they are on GPRs. + if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) { + SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); + MIRBuilder.buildCopy({SrcReg}, {I.getOperand(2)}); + + // Make sure the copy ends up getting constrained properly. + RBI.constrainGenericRegister(I.getOperand(2).getReg(), + AArch64::GPR32RegClass, MRI); + } + + if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) + DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); + + // Actually insert the instruction. + auto SHA1Inst = MIRBuilder.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg}); + constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI); + + // Did we create a new register for the destination? + if (DstReg != I.getOperand(0).getReg()) { + // Yep. Copy the result of the instruction back into the original + // destination. + MIRBuilder.buildCopy({I.getOperand(0)}, {DstReg}); + RBI.constrainGenericRegister(I.getOperand(0).getReg(), + AArch64::GPR32RegClass, MRI); + } + + I.eraseFromParent(); + return true; + } + return false; +} + +static Optional<uint64_t> getImmedFromMO(const MachineOperand &Root) { + auto &MI = *Root.getParent(); + auto &MBB = *MI.getParent(); + auto &MF = *MBB.getParent(); + auto &MRI = MF.getRegInfo(); uint64_t Immed; if (Root.isImm()) Immed = Root.getImm(); else if (Root.isCImm()) Immed = Root.getCImm()->getZExtValue(); else if (Root.isReg()) { - MachineInstr *Def = MRI.getVRegDef(Root.getReg()); - if (Def->getOpcode() != TargetOpcode::G_CONSTANT) - return None; - MachineOperand &Op1 = Def->getOperand(1); - if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64) + auto ValAndVReg = + getConstantVRegValWithLookThrough(Root.getReg(), MRI, true); + if (!ValAndVReg) return None; - Immed = Op1.getCImm()->getZExtValue(); + Immed = ValAndVReg->Value; } else return None; + return Immed; +} +InstructionSelector::ComplexRendererFns +AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const { + auto MaybeImmed = getImmedFromMO(Root); + if (MaybeImmed == None || *MaybeImmed > 31) + return None; + uint64_t Enc = (32 - *MaybeImmed) & 0x1f; + return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; +} + +InstructionSelector::ComplexRendererFns +AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const { + auto MaybeImmed = getImmedFromMO(Root); + if (MaybeImmed == None || *MaybeImmed > 31) + return None; + uint64_t Enc = 31 - *MaybeImmed; + return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; +} + +InstructionSelector::ComplexRendererFns +AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const { + auto MaybeImmed = getImmedFromMO(Root); + if (MaybeImmed == None || *MaybeImmed > 63) + return None; + uint64_t Enc = (64 - *MaybeImmed) & 0x3f; + return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; +} + +InstructionSelector::ComplexRendererFns +AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const { + auto MaybeImmed = getImmedFromMO(Root); + if (MaybeImmed == None || *MaybeImmed > 63) + return None; + uint64_t Enc = 63 - *MaybeImmed; + return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; +} + +/// SelectArithImmed - Select an immediate value that can be represented as +/// a 12-bit value shifted left by either 0 or 12. If so, return true with +/// Val set to the 12-bit value and Shift set to the shifter operand. +InstructionSelector::ComplexRendererFns +AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const { + // This function is called from the addsub_shifted_imm ComplexPattern, + // which lists [imm] as the list of opcode it's interested in, however + // we still need to check whether the operand is actually an immediate + // here because the ComplexPattern opcode list is only used in + // root-level opcode matching. + auto MaybeImmed = getImmedFromMO(Root); + if (MaybeImmed == None) + return None; + uint64_t Immed = *MaybeImmed; unsigned ShiftAmt; if (Immed >> 12 == 0) { diff --git a/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/lib/Target/AArch64/AArch64LegalizerInfo.cpp index 6f7fb7a8bc21..a985b330eafa 100644 --- a/lib/Target/AArch64/AArch64LegalizerInfo.cpp +++ b/lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -1,9 +1,8 @@ //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -22,8 +21,11 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Type.h" +#define DEBUG_TYPE "aarch64-legalinfo" + using namespace llvm; using namespace LegalizeActions; +using namespace LegalizeMutations; using namespace LegalityPredicates; AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { @@ -46,9 +48,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { const LLT v2s32 = LLT::vector(2, 32); const LLT v4s32 = LLT::vector(4, 32); const LLT v2s64 = LLT::vector(2, 64); + const LLT v2p0 = LLT::vector(2, p0); getActionDefinitionsBuilder(G_IMPLICIT_DEF) - .legalFor({p0, s1, s8, s16, s32, s64, v2s64}) + .legalFor({p0, s1, s8, s16, s32, s64, v4s32, v2s64}) .clampScalar(0, s1, s64) .widenScalarToNextPow2(0, 8) .fewerElementsIf( @@ -65,33 +68,58 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { }); getActionDefinitionsBuilder(G_PHI) - .legalFor({p0, s16, s32, s64}) + .legalFor({p0, s16, s32, s64, v2s32, v4s32, v2s64}) .clampScalar(0, s16, s64) .widenScalarToNextPow2(0); getActionDefinitionsBuilder(G_BSWAP) - .legalFor({s32, s64}) + .legalFor({s32, s64, v4s32, v2s32, v2s64}) .clampScalar(0, s16, s64) .widenScalarToNextPow2(0); - getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR, G_SHL}) - .legalFor({s32, s64, v2s32, v4s32, v2s64}) + getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR}) + .legalFor({s32, s64, v2s32, v4s32, v2s64, v8s16, v16s8}) .clampScalar(0, s32, s64) .widenScalarToNextPow2(0) .clampNumElements(0, v2s32, v4s32) .clampNumElements(0, v2s64, v2s64) .moreElementsToNextPow2(0); + getActionDefinitionsBuilder(G_SHL) + .legalFor({{s32, s32}, {s64, s64}, + {v2s32, v2s32}, {v4s32, v4s32}, {v2s64, v2s64}}) + .clampScalar(1, s32, s64) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0) + .clampNumElements(0, v2s32, v4s32) + .clampNumElements(0, v2s64, v2s64) + .moreElementsToNextPow2(0) + .minScalarSameAs(1, 0); + getActionDefinitionsBuilder(G_GEP) .legalFor({{p0, s64}}) .clampScalar(1, s64, s64); getActionDefinitionsBuilder(G_PTR_MASK).legalFor({p0}); - getActionDefinitionsBuilder({G_LSHR, G_ASHR, G_SDIV, G_UDIV}) + getActionDefinitionsBuilder({G_SDIV, G_UDIV}) .legalFor({s32, s64}) .clampScalar(0, s32, s64) - .widenScalarToNextPow2(0); + .widenScalarToNextPow2(0) + .scalarize(0); + + getActionDefinitionsBuilder({G_LSHR, G_ASHR}) + .customIf([=](const LegalityQuery &Query) { + const auto &SrcTy = Query.Types[0]; + const auto &AmtTy = Query.Types[1]; + return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 && + AmtTy.getSizeInBits() == 32; + }) + .legalFor( + {{s32, s32}, {s32, s64}, {s64, s64}, {v2s32, v2s32}, {v4s32, v4s32}}) + .clampScalar(1, s32, s64) + .clampScalar(0, s32, s64) + .minScalarSameAs(1, 0); getActionDefinitionsBuilder({G_SREM, G_UREM}) .lowerFor({s1, s8, s16, s32, s64}); @@ -101,15 +129,26 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64}); - getActionDefinitionsBuilder({G_UADDE, G_USUBE, G_SADDO, G_SSUBO}) + getActionDefinitionsBuilder({G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO}) .legalFor({{s32, s1}, {s64, s1}}); - getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMA, G_FMUL, G_FDIV}) - .legalFor({s32, s64}); + getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FNEG}) + .legalFor({s32, s64, v2s64, v4s32, v2s32}); - getActionDefinitionsBuilder({G_FREM, G_FPOW}).libcallFor({s32, s64}); + getActionDefinitionsBuilder(G_FREM).libcallFor({s32, s64}); - getActionDefinitionsBuilder(G_FCEIL) + getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR, G_FRINT, + G_FMA, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, + G_FNEARBYINT}) + // If we don't have full FP16 support, then scalarize the elements of + // vectors containing fp16 types. + .fewerElementsIf( + [=, &ST](const LegalityQuery &Query) { + const auto &Ty = Query.Types[0]; + return Ty.isVector() && Ty.getElementType() == s16 && + !ST.hasFullFP16(); + }, + [=](const LegalityQuery &Query) { return std::make_pair(0, s16); }) // If we don't have full FP16 support, then widen s16 to s32 if we // encounter it. .widenScalarIf( @@ -117,7 +156,15 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { return Query.Types[0] == s16 && !ST.hasFullFP16(); }, [=](const LegalityQuery &Query) { return std::make_pair(0, s32); }) - .legalFor({s16, s32, s64, v2s32, v4s32, v2s64}); + .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16}); + + getActionDefinitionsBuilder( + {G_FCOS, G_FSIN, G_FLOG10, G_FLOG, G_FLOG2, G_FEXP, G_FEXP2, G_FPOW}) + // We need a call for these, so we always need to scalarize. + .scalarize(0) + // Regardless of FP16 support, widen 16-bit elements to 32-bits. + .minScalar(0, s32) + .libcallFor({s32, s64, v2s32, v4s32, v2s64}); getActionDefinitionsBuilder(G_INSERT) .unsupportedIf([=](const LegalityQuery &Query) { @@ -158,12 +205,15 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { .widenScalarToNextPow2(0); getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD}) - .legalForTypesWithMemSize({{s32, p0, 8}, - {s32, p0, 16}, - {s32, p0, 32}, - {s64, p0, 64}, - {p0, p0, 64}, - {v2s32, p0, 64}}) + .legalForTypesWithMemDesc({{s32, p0, 8, 8}, + {s32, p0, 16, 8}, + {s32, p0, 32, 8}, + {s64, p0, 8, 2}, + {s64, p0, 16, 2}, + {s64, p0, 32, 4}, + {s64, p0, 64, 8}, + {p0, p0, 64, 8}, + {v2s32, p0, 64, 8}}) .clampScalar(0, s32, s64) .widenScalarToNextPow2(0) // TODO: We could support sum-of-pow2's but the lowering code doesn't know @@ -172,16 +222,30 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { // Lower anything left over into G_*EXT and G_LOAD .lower(); + auto IsPtrVecPred = [=](const LegalityQuery &Query) { + const LLT &ValTy = Query.Types[0]; + if (!ValTy.isVector()) + return false; + const LLT EltTy = ValTy.getElementType(); + return EltTy.isPointer() && EltTy.getAddressSpace() == 0; + }; + getActionDefinitionsBuilder(G_LOAD) - .legalForTypesWithMemSize({{s8, p0, 8}, - {s16, p0, 16}, - {s32, p0, 32}, - {s64, p0, 64}, - {p0, p0, 64}, - {v2s32, p0, 64}}) + .legalForTypesWithMemDesc({{s8, p0, 8, 8}, + {s16, p0, 16, 8}, + {s32, p0, 32, 8}, + {s64, p0, 64, 8}, + {p0, p0, 64, 8}, + {v8s8, p0, 64, 8}, + {v16s8, p0, 128, 8}, + {v4s16, p0, 64, 8}, + {v8s16, p0, 128, 8}, + {v2s32, p0, 64, 8}, + {v4s32, p0, 128, 8}, + {v2s64, p0, 128, 8}}) // These extends are also legal - .legalForTypesWithMemSize({{s32, p0, 8}, - {s32, p0, 16}}) + .legalForTypesWithMemDesc({{s32, p0, 8, 8}, + {s32, p0, 16, 8}}) .clampScalar(0, s8, s64) .widenScalarToNextPow2(0) // TODO: We could support sum-of-pow2's but the lowering code doesn't know @@ -191,16 +255,22 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { .lowerIf([=](const LegalityQuery &Query) { return Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits; }) - .clampNumElements(0, v2s32, v2s32) - .clampMaxNumElements(0, s64, 1); + .clampMaxNumElements(0, s32, 2) + .clampMaxNumElements(0, s64, 1) + .customIf(IsPtrVecPred); getActionDefinitionsBuilder(G_STORE) - .legalForTypesWithMemSize({{s8, p0, 8}, - {s16, p0, 16}, - {s32, p0, 32}, - {s64, p0, 64}, - {p0, p0, 64}, - {v2s32, p0, 64}}) + .legalForTypesWithMemDesc({{s8, p0, 8, 8}, + {s16, p0, 16, 8}, + {s32, p0, 32, 8}, + {s64, p0, 64, 8}, + {p0, p0, 64, 8}, + {v16s8, p0, 128, 8}, + {v4s16, p0, 64, 8}, + {v8s16, p0, 128, 8}, + {v2s32, p0, 64, 8}, + {v4s32, p0, 128, 8}, + {v2s64, p0, 128, 8}}) .clampScalar(0, s8, s64) .widenScalarToNextPow2(0) // TODO: We could support sum-of-pow2's but the lowering code doesn't know @@ -210,23 +280,48 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { return Query.Types[0].isScalar() && Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits; }) - .clampNumElements(0, v2s32, v2s32) - .clampMaxNumElements(0, s64, 1); + .clampMaxNumElements(0, s32, 2) + .clampMaxNumElements(0, s64, 1) + .customIf(IsPtrVecPred); // Constants getActionDefinitionsBuilder(G_CONSTANT) - .legalFor({p0, s32, s64}) - .clampScalar(0, s32, s64) + .legalFor({p0, s8, s16, s32, s64}) + .clampScalar(0, s8, s64) .widenScalarToNextPow2(0); getActionDefinitionsBuilder(G_FCONSTANT) .legalFor({s32, s64}) .clampScalar(0, s32, s64); getActionDefinitionsBuilder(G_ICMP) - .legalFor({{s32, s32}, {s32, s64}, {s32, p0}}) + .legalFor({{s32, s32}, + {s32, s64}, + {s32, p0}, + {v4s32, v4s32}, + {v2s32, v2s32}, + {v2s64, v2s64}, + {v2s64, v2p0}, + {v4s16, v4s16}, + {v8s16, v8s16}, + {v8s8, v8s8}, + {v16s8, v16s8}}) .clampScalar(0, s32, s32) .clampScalar(1, s32, s64) - .widenScalarToNextPow2(1); + .minScalarEltSameAsIf( + [=](const LegalityQuery &Query) { + const LLT &Ty = Query.Types[0]; + const LLT &SrcTy = Query.Types[1]; + return Ty.isVector() && !SrcTy.getElementType().isPointer() && + Ty.getElementType() != SrcTy.getElementType(); + }, + 0, 1) + .minScalarOrEltIf( + [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; }, + 1, s32) + .minScalarOrEltIf( + [=](const LegalityQuery &Query) { return Query.Types[1] == v2p0; }, 0, + s64) + .widenScalarOrEltToNextPow2(1); getActionDefinitionsBuilder(G_FCMP) .legalFor({{s32, s32}, {s32, s64}}) @@ -236,24 +331,48 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { // Extensions getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT}) - .legalForCartesianProduct({s8, s16, s32, s64}, {s1, s8, s16, s32}); + .legalIf([=](const LegalityQuery &Query) { + unsigned DstSize = Query.Types[0].getSizeInBits(); + + // Make sure that we have something that will fit in a register, and + // make sure it's a power of 2. + if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize)) + return false; + + const LLT &SrcTy = Query.Types[1]; + + // Special case for s1. + if (SrcTy == s1) + return true; + + // Make sure we fit in a register otherwise. Don't bother checking that + // the source type is below 128 bits. We shouldn't be allowing anything + // through which is wider than the destination in the first place. + unsigned SrcSize = SrcTy.getSizeInBits(); + if (SrcSize < 8 || !isPowerOf2_32(SrcSize)) + return false; + + return true; + }); + + getActionDefinitionsBuilder(G_TRUNC).alwaysLegal(); // FP conversions getActionDefinitionsBuilder(G_FPTRUNC).legalFor( - {{s16, s32}, {s16, s64}, {s32, s64}}); + {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}}); getActionDefinitionsBuilder(G_FPEXT).legalFor( - {{s32, s16}, {s64, s16}, {s64, s32}}); + {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}}); // Conversions getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI}) - .legalForCartesianProduct({s32, s64}) + .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32}) .clampScalar(0, s32, s64) .widenScalarToNextPow2(0) .clampScalar(1, s32, s64) .widenScalarToNextPow2(1); getActionDefinitionsBuilder({G_SITOFP, G_UITOFP}) - .legalForCartesianProduct({s32, s64}) + .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32}) .clampScalar(1, s32, s64) .widenScalarToNextPow2(1) .clampScalar(0, s32, s64) @@ -264,10 +383,13 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0}); // Select + // FIXME: We can probably do a bit better than just scalarizing vector + // selects. getActionDefinitionsBuilder(G_SELECT) .legalFor({{s32, s1}, {s64, s1}, {p0, s1}}) .clampScalar(0, s32, s64) - .widenScalarToNextPow2(0); + .widenScalarToNextPow2(0) + .scalarize(0); // Pointer-handling getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0}); @@ -291,7 +413,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { // number of bits but it's what the previous code described and fixing // it breaks tests. .legalForCartesianProduct({s1, s8, s16, s32, s64, s128, v16s8, v8s8, v4s8, - v8s16, v4s16, v2s16, v4s32, v2s32, v2s64}); + v8s16, v4s16, v2s16, v4s32, v2s32, v2s64, + v2p0}); getActionDefinitionsBuilder(G_VASTART).legalFor({p0}); @@ -335,11 +458,6 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { } return false; }; - auto scalarize = - [](const LegalityQuery &Query, unsigned TypeIdx) { - const LLT &Ty = Query.Types[TypeIdx]; - return std::make_pair(TypeIdx, Ty.getElementType()); - }; // FIXME: This rule is horrible, but specifies the same as what we had // before with the particularly strange definitions removed (e.g. @@ -353,10 +471,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { // Break up vectors with weird elements into scalars .fewerElementsIf( [=](const LegalityQuery &Query) { return notValidElt(Query, 0); }, - [=](const LegalityQuery &Query) { return scalarize(Query, 0); }) + scalarize(0)) .fewerElementsIf( [=](const LegalityQuery &Query) { return notValidElt(Query, 1); }, - [=](const LegalityQuery &Query) { return scalarize(Query, 1); }) + scalarize(1)) // Clamp the big scalar to s8-s512 and make it either a power of 2, 192, // or 384. .clampScalar(BigTyIdx, s8, s512) @@ -397,16 +515,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { return BigTy.getSizeInBits() % LitTy.getSizeInBits() == 0; }) // Any vectors left are the wrong size. Scalarize them. - .fewerElementsIf([](const LegalityQuery &Query) { return true; }, - [](const LegalityQuery &Query) { - return std::make_pair( - 0, Query.Types[0].getElementType()); - }) - .fewerElementsIf([](const LegalityQuery &Query) { return true; }, - [](const LegalityQuery &Query) { - return std::make_pair( - 1, Query.Types[1].getElementType()); - }); + .scalarize(0) + .scalarize(1); } getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT) @@ -417,11 +527,24 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { .minScalar(2, s64) .legalIf([=](const LegalityQuery &Query) { const LLT &VecTy = Query.Types[1]; - return VecTy == v4s32 || VecTy == v2s64; + return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 || + VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32; + }); + + getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT) + .legalIf([=](const LegalityQuery &Query) { + const LLT &VecTy = Query.Types[0]; + // TODO: Support s8 and s16 + return VecTy == v2s32 || VecTy == v4s32 || VecTy == v2s64; }); getActionDefinitionsBuilder(G_BUILD_VECTOR) - .legalFor({{v4s32, s32}, {v2s64, s64}}) + .legalFor({{v4s16, s16}, + {v8s16, s16}, + {v2s32, s32}, + {v4s32, s32}, + {v2p0, p0}, + {v2s64, s64}}) .clampNumElements(0, v4s32, v4s32) .clampNumElements(0, v2s64, v2s64) @@ -432,6 +555,42 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { }) .minScalarSameAs(1, 0); + getActionDefinitionsBuilder(G_CTLZ).legalForCartesianProduct( + {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32}) + .scalarize(1); + + getActionDefinitionsBuilder(G_SHUFFLE_VECTOR) + .legalIf([=](const LegalityQuery &Query) { + const LLT &DstTy = Query.Types[0]; + const LLT &SrcTy = Query.Types[1]; + // For now just support the TBL2 variant which needs the source vectors + // to be the same size as the dest. + if (DstTy != SrcTy) + return false; + for (auto &Ty : {v2s32, v4s32, v2s64}) { + if (DstTy == Ty) + return true; + } + return false; + }) + // G_SHUFFLE_VECTOR can have scalar sources (from 1 x s vectors), we + // just want those lowered into G_BUILD_VECTOR + .lowerIf([=](const LegalityQuery &Query) { + return !Query.Types[1].isVector(); + }) + .clampNumElements(0, v4s32, v4s32) + .clampNumElements(0, v2s64, v2s64); + + getActionDefinitionsBuilder(G_CONCAT_VECTORS) + .legalFor({{v4s32, v2s32}, {v8s16, v4s16}}); + + getActionDefinitionsBuilder(G_JUMP_TABLE) + .legalFor({{p0}, {s64}}); + + getActionDefinitionsBuilder(G_BRJT).legalIf([=](const LegalityQuery &Query) { + return Query.Types[0] == p0 && Query.Types[1] == s64; + }); + computeTables(); verify(*ST.getInstrInfo()); } @@ -446,37 +605,106 @@ bool AArch64LegalizerInfo::legalizeCustom(MachineInstr &MI, return false; case TargetOpcode::G_VAARG: return legalizeVaArg(MI, MRI, MIRBuilder); + case TargetOpcode::G_LOAD: + case TargetOpcode::G_STORE: + return legalizeLoadStore(MI, MRI, MIRBuilder, Observer); + case TargetOpcode::G_SHL: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: + return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer); } llvm_unreachable("expected switch to return"); } +bool AArch64LegalizerInfo::legalizeShlAshrLshr( + MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, + GISelChangeObserver &Observer) const { + assert(MI.getOpcode() == TargetOpcode::G_ASHR || + MI.getOpcode() == TargetOpcode::G_LSHR || + MI.getOpcode() == TargetOpcode::G_SHL); + // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the + // imported patterns can select it later. Either way, it will be legal. + Register AmtReg = MI.getOperand(2).getReg(); + auto *CstMI = MRI.getVRegDef(AmtReg); + assert(CstMI && "expected to find a vreg def"); + if (CstMI->getOpcode() != TargetOpcode::G_CONSTANT) + return true; + // Check the shift amount is in range for an immediate form. + unsigned Amount = CstMI->getOperand(1).getCImm()->getZExtValue(); + if (Amount > 31) + return true; // This will have to remain a register variant. + assert(MRI.getType(AmtReg).getSizeInBits() == 32); + MIRBuilder.setInstr(MI); + auto ExtCst = MIRBuilder.buildZExt(LLT::scalar(64), AmtReg); + MI.getOperand(2).setReg(ExtCst.getReg(0)); + return true; +} + +bool AArch64LegalizerInfo::legalizeLoadStore( + MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, + GISelChangeObserver &Observer) const { + assert(MI.getOpcode() == TargetOpcode::G_STORE || + MI.getOpcode() == TargetOpcode::G_LOAD); + // Here we just try to handle vector loads/stores where our value type might + // have pointer elements, which the SelectionDAG importer can't handle. To + // allow the existing patterns for s64 to fire for p0, we just try to bitcast + // the value to use s64 types. + + // Custom legalization requires the instruction, if not deleted, must be fully + // legalized. In order to allow further legalization of the inst, we create + // a new instruction and erase the existing one. + + unsigned ValReg = MI.getOperand(0).getReg(); + const LLT ValTy = MRI.getType(ValReg); + + if (!ValTy.isVector() || !ValTy.getElementType().isPointer() || + ValTy.getElementType().getAddressSpace() != 0) { + LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store"); + return false; + } + + MIRBuilder.setInstr(MI); + unsigned PtrSize = ValTy.getElementType().getSizeInBits(); + const LLT NewTy = LLT::vector(ValTy.getNumElements(), PtrSize); + auto &MMO = **MI.memoperands_begin(); + if (MI.getOpcode() == TargetOpcode::G_STORE) { + auto Bitcast = MIRBuilder.buildBitcast({NewTy}, {ValReg}); + MIRBuilder.buildStore(Bitcast.getReg(0), MI.getOperand(1).getReg(), MMO); + } else { + unsigned NewReg = MRI.createGenericVirtualRegister(NewTy); + auto NewLoad = MIRBuilder.buildLoad(NewReg, MI.getOperand(1).getReg(), MMO); + MIRBuilder.buildBitcast({ValReg}, {NewLoad}); + } + MI.eraseFromParent(); + return true; +} + bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const { MIRBuilder.setInstr(MI); MachineFunction &MF = MIRBuilder.getMF(); unsigned Align = MI.getOperand(2).getImm(); - unsigned Dst = MI.getOperand(0).getReg(); - unsigned ListPtr = MI.getOperand(1).getReg(); + Register Dst = MI.getOperand(0).getReg(); + Register ListPtr = MI.getOperand(1).getReg(); LLT PtrTy = MRI.getType(ListPtr); LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); const unsigned PtrSize = PtrTy.getSizeInBits() / 8; - unsigned List = MRI.createGenericVirtualRegister(PtrTy); + Register List = MRI.createGenericVirtualRegister(PtrTy); MIRBuilder.buildLoad( List, ListPtr, *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, PtrSize, /* Align = */ PtrSize)); - unsigned DstPtr; + Register DstPtr; if (Align > PtrSize) { // Realign the list to the actual required alignment. auto AlignMinus1 = MIRBuilder.buildConstant(IntPtrTy, Align - 1); - unsigned ListTmp = MRI.createGenericVirtualRegister(PtrTy); - MIRBuilder.buildGEP(ListTmp, List, AlignMinus1->getOperand(0).getReg()); + auto ListTmp = MIRBuilder.buildGEP(PtrTy, List, AlignMinus1.getReg(0)); DstPtr = MRI.createGenericVirtualRegister(PtrTy); MIRBuilder.buildPtrMask(DstPtr, ListTmp, Log2_64(Align)); @@ -489,11 +717,9 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, ValSize, std::max(Align, PtrSize))); - unsigned SizeReg = MRI.createGenericVirtualRegister(IntPtrTy); - MIRBuilder.buildConstant(SizeReg, alignTo(ValSize, PtrSize)); + auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrSize)); - unsigned NewList = MRI.createGenericVirtualRegister(PtrTy); - MIRBuilder.buildGEP(NewList, DstPtr, SizeReg); + auto NewList = MIRBuilder.buildGEP(PtrTy, DstPtr, Size.getReg(0)); MIRBuilder.buildStore( NewList, ListPtr, diff --git a/lib/Target/AArch64/AArch64LegalizerInfo.h b/lib/Target/AArch64/AArch64LegalizerInfo.h index 77e8bdc7623c..f3362a18620f 100644 --- a/lib/Target/AArch64/AArch64LegalizerInfo.h +++ b/lib/Target/AArch64/AArch64LegalizerInfo.h @@ -1,9 +1,8 @@ //===- AArch64LegalizerInfo --------------------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -35,6 +34,12 @@ public: private: bool legalizeVaArg(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const; + bool legalizeLoadStore(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder, + GISelChangeObserver &Observer) const; + bool legalizeShlAshrLshr(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder, + GISelChangeObserver &Observer) const; }; } // End llvm namespace. #endif diff --git a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp index aa732a99469c..65b5f906e3f6 100644 --- a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -1,9 +1,8 @@ //===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -934,8 +933,6 @@ AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI, ? getLdStOffsetOp(*StoreI).getImm() : getLdStOffsetOp(*StoreI).getImm() * StoreSize; int Width = LoadSize * 8; - int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset); - int Imms = Immr + Width - 1; unsigned DestReg = IsStoreXReg ? TRI->getMatchingSuperReg(LdRt, AArch64::sub_32, &AArch64::GPR64RegClass) @@ -945,8 +942,8 @@ AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI, (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) && "Invalid offset"); - Immr = 8 * (UnscaledLdOffset - UnscaledStOffset); - Imms = Immr + Width - 1; + int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset); + int Imms = Immr + Width - 1; if (UnscaledLdOffset == UnscaledStOffset) { uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N | ((Immr) << 6) // immr diff --git a/lib/Target/AArch64/AArch64MCInstLower.cpp b/lib/Target/AArch64/AArch64MCInstLower.cpp index d71359223b1b..e7d4a2789a28 100644 --- a/lib/Target/AArch64/AArch64MCInstLower.cpp +++ b/lib/Target/AArch64/AArch64MCInstLower.cpp @@ -1,9 +1,8 @@ //==-- AArch64MCInstLower.cpp - Convert AArch64 MachineInstr to an MCInst --==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64MCInstLower.h b/lib/Target/AArch64/AArch64MCInstLower.h index aa30fe1fa707..8f3148a98410 100644 --- a/lib/Target/AArch64/AArch64MCInstLower.h +++ b/lib/Target/AArch64/AArch64MCInstLower.h @@ -1,9 +1,8 @@ //===-- AArch64MCInstLower.h - Lower MachineInstr to MCInst ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/lib/Target/AArch64/AArch64MachineFunctionInfo.h index 5183e7d3c0d0..0efeeb272ec1 100644 --- a/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -1,9 +1,8 @@ //=- AArch64MachineFunctionInfo.h - AArch64 machine function info -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -92,6 +91,11 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { /// other stack allocations. bool CalleeSaveStackHasFreeSpace = false; + /// SRetReturnReg - sret lowering includes returning the value of the + /// returned struct in a register. This field holds the virtual register into + /// which the sret argument is passed. + unsigned SRetReturnReg = 0; + /// Has a value when it is known whether or not the function uses a /// redzone, and no value otherwise. /// Initialized during frame lowering, unless the function has the noredzone @@ -101,6 +105,12 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { /// ForwardedMustTailRegParms - A list of virtual and physical registers /// that must be forwarded to every musttail call. SmallVector<ForwardedRegister, 1> ForwardedMustTailRegParms; + + // Offset from SP-at-entry to the tagged base pointer. + // Tagged base pointer is set up to point to the first (lowest address) tagged + // stack slot. + unsigned TaggedBasePointerOffset; + public: AArch64FunctionInfo() = default; @@ -166,6 +176,9 @@ public: unsigned getVarArgsFPRSize() const { return VarArgsFPRSize; } void setVarArgsFPRSize(unsigned Size) { VarArgsFPRSize = Size; } + unsigned getSRetReturnReg() const { return SRetReturnReg; } + void setSRetReturnReg(unsigned Reg) { SRetReturnReg = Reg; } + unsigned getJumpTableEntrySize(int Idx) const { auto It = JumpTableEntryInfo.find(Idx); if (It != JumpTableEntryInfo.end()) @@ -217,6 +230,13 @@ public: return ForwardedMustTailRegParms; } + unsigned getTaggedBasePointerOffset() const { + return TaggedBasePointerOffset; + } + void setTaggedBasePointerOffset(unsigned Offset) { + TaggedBasePointerOffset = Offset; + } + private: // Hold the lists of LOHs. MILOHContainer LOHContainerSet; diff --git a/lib/Target/AArch64/AArch64MacroFusion.cpp b/lib/Target/AArch64/AArch64MacroFusion.cpp index bc596dd38b6e..9a2103579a6a 100644 --- a/lib/Target/AArch64/AArch64MacroFusion.cpp +++ b/lib/Target/AArch64/AArch64MacroFusion.cpp @@ -1,9 +1,8 @@ //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64MacroFusion.h b/lib/Target/AArch64/AArch64MacroFusion.h index 32d90d4c40d6..4e7ccbe4baab 100644 --- a/lib/Target/AArch64/AArch64MacroFusion.h +++ b/lib/Target/AArch64/AArch64MacroFusion.h @@ -1,9 +1,8 @@ //===- AArch64MacroFusion.h - AArch64 Macro Fusion ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp b/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp index ccf646575296..aff861aae6be 100644 --- a/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp +++ b/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp @@ -1,9 +1,8 @@ //===-- AArch64PBQPRegAlloc.cpp - AArch64 specific PBQP constraints -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file contains the AArch64 / Cortex-A57 specific register allocation diff --git a/lib/Target/AArch64/AArch64PBQPRegAlloc.h b/lib/Target/AArch64/AArch64PBQPRegAlloc.h index b99c1d1d6b3e..5ea91b4a1967 100644 --- a/lib/Target/AArch64/AArch64PBQPRegAlloc.h +++ b/lib/Target/AArch64/AArch64PBQPRegAlloc.h @@ -1,9 +1,8 @@ //==- AArch64PBQPRegAlloc.h - AArch64 specific PBQP constraints --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64PerfectShuffle.h b/lib/Target/AArch64/AArch64PerfectShuffle.h index 9e9eec48c555..f443cd03935c 100644 --- a/lib/Target/AArch64/AArch64PerfectShuffle.h +++ b/lib/Target/AArch64/AArch64PerfectShuffle.h @@ -1,9 +1,8 @@ //===-- AArch64PerfectShuffle.h - AdvSIMD Perfect Shuffle Table -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64PfmCounters.td b/lib/Target/AArch64/AArch64PfmCounters.td index 16ba3e4282a0..b1d1664e3f1b 100644 --- a/lib/Target/AArch64/AArch64PfmCounters.td +++ b/lib/Target/AArch64/AArch64PfmCounters.td @@ -1,9 +1,8 @@ //===-- AArch64PfmCounters.td - AArch64 Hardware Counters --*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp b/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp index 3da9306e6460..5f7245bfbd74 100644 --- a/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp +++ b/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp @@ -1,9 +1,8 @@ //=== lib/CodeGen/GlobalISel/AArch64PreLegalizerCombiner.cpp --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -44,6 +43,10 @@ bool AArch64PreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer, switch (MI.getOpcode()) { default: return false; + case TargetOpcode::COPY: + return Helper.tryCombineCopy(MI); + case TargetOpcode::G_BR: + return Helper.tryCombineBr(MI); case TargetOpcode::G_LOAD: case TargetOpcode::G_SEXTLOAD: case TargetOpcode::G_ZEXTLOAD: diff --git a/lib/Target/AArch64/AArch64PromoteConstant.cpp b/lib/Target/AArch64/AArch64PromoteConstant.cpp index 01d8a35bbc23..a594ecb71fc9 100644 --- a/lib/Target/AArch64/AArch64PromoteConstant.cpp +++ b/lib/Target/AArch64/AArch64PromoteConstant.cpp @@ -1,9 +1,8 @@ //==- AArch64PromoteConstant.cpp - Promote constant to global for AArch64 --==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -494,7 +493,8 @@ void AArch64PromoteConstant::insertDefinitions(Function &F, for (const auto &IPI : InsertPts) { // Create the load of the global variable. IRBuilder<> Builder(IPI.first); - LoadInst *LoadedCst = Builder.CreateLoad(&PromotedGV); + LoadInst *LoadedCst = + Builder.CreateLoad(PromotedGV.getValueType(), &PromotedGV); LLVM_DEBUG(dbgs() << "**********\n"); LLVM_DEBUG(dbgs() << "New def: "); LLVM_DEBUG(LoadedCst->print(dbgs())); diff --git a/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp b/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp index fcb0b36a9f6d..0d75ab7ac8a9 100644 --- a/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp +++ b/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp @@ -1,9 +1,8 @@ //=- AArch64RedundantCopyElimination.cpp - Remove useless copy for AArch64 -=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // This pass removes unnecessary copies/moves in BBs based on a dominating // condition. @@ -380,8 +379,8 @@ bool AArch64RedundantCopyElimination::optimizeBlock(MachineBasicBlock *MBB) { bool IsCopy = MI->isCopy(); bool IsMoveImm = MI->isMoveImmediate(); if (IsCopy || IsMoveImm) { - MCPhysReg DefReg = MI->getOperand(0).getReg(); - MCPhysReg SrcReg = IsCopy ? MI->getOperand(1).getReg() : 0; + Register DefReg = MI->getOperand(0).getReg(); + Register SrcReg = IsCopy ? MI->getOperand(1).getReg() : Register(); int64_t SrcImm = IsMoveImm ? MI->getOperand(1).getImm() : 0; if (!MRI->isReserved(DefReg) && ((IsCopy && (SrcReg == AArch64::XZR || SrcReg == AArch64::WZR)) || diff --git a/lib/Target/AArch64/AArch64RegisterBankInfo.cpp b/lib/Target/AArch64/AArch64RegisterBankInfo.cpp index 68c48a5ec216..b52259cc9acd 100644 --- a/lib/Target/AArch64/AArch64RegisterBankInfo.cpp +++ b/lib/Target/AArch64/AArch64RegisterBankInfo.cpp @@ -1,9 +1,8 @@ //===- AArch64RegisterBankInfo.cpp ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -243,12 +242,17 @@ const RegisterBank &AArch64RegisterBankInfo::getRegBankFromRegClass( case AArch64::GPR32RegClassID: case AArch64::GPR32spRegClassID: case AArch64::GPR32sponlyRegClassID: + case AArch64::GPR32argRegClassID: case AArch64::GPR32allRegClassID: case AArch64::GPR64commonRegClassID: case AArch64::GPR64RegClassID: case AArch64::GPR64spRegClassID: case AArch64::GPR64sponlyRegClassID: + case AArch64::GPR64argRegClassID: case AArch64::GPR64allRegClassID: + case AArch64::GPR64noipRegClassID: + case AArch64::GPR64common_and_GPR64noipRegClassID: + case AArch64::GPR64noip_and_tcGPR64RegClassID: case AArch64::tcGPR64RegClassID: case AArch64::WSeqPairsClassRegClassID: case AArch64::XSeqPairsClassRegClassID: @@ -385,11 +389,26 @@ static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) { case TargetOpcode::G_FADD: case TargetOpcode::G_FSUB: case TargetOpcode::G_FMUL: + case TargetOpcode::G_FMA: case TargetOpcode::G_FDIV: case TargetOpcode::G_FCONSTANT: case TargetOpcode::G_FPEXT: case TargetOpcode::G_FPTRUNC: case TargetOpcode::G_FCEIL: + case TargetOpcode::G_FFLOOR: + case TargetOpcode::G_FNEARBYINT: + case TargetOpcode::G_FNEG: + case TargetOpcode::G_FCOS: + case TargetOpcode::G_FSIN: + case TargetOpcode::G_FLOG10: + case TargetOpcode::G_FLOG: + case TargetOpcode::G_FLOG2: + case TargetOpcode::G_FSQRT: + case TargetOpcode::G_FABS: + case TargetOpcode::G_FEXP: + case TargetOpcode::G_FRINT: + case TargetOpcode::G_INTRINSIC_TRUNC: + case TargetOpcode::G_INTRINSIC_ROUND: return true; } return false; @@ -438,6 +457,54 @@ AArch64RegisterBankInfo::getSameKindOfOperandsMapping( getValueMapping(RBIdx, Size), NumOperands); } +bool AArch64RegisterBankInfo::hasFPConstraints( + const MachineInstr &MI, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const { + unsigned Op = MI.getOpcode(); + + // Do we have an explicit floating point instruction? + if (isPreISelGenericFloatingPointOpcode(Op)) + return true; + + // No. Check if we have a copy-like instruction. If we do, then we could + // still be fed by floating point instructions. + if (Op != TargetOpcode::COPY && !MI.isPHI()) + return false; + + // MI is copy-like. Return true if it outputs an FPR. + return getRegBank(MI.getOperand(0).getReg(), MRI, TRI) == + &AArch64::FPRRegBank; +} + +bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI, + const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const { + switch (MI.getOpcode()) { + case TargetOpcode::G_FPTOSI: + case TargetOpcode::G_FPTOUI: + case TargetOpcode::G_FCMP: + return true; + default: + break; + } + return hasFPConstraints(MI, MRI, TRI); +} + +bool AArch64RegisterBankInfo::onlyDefinesFP( + const MachineInstr &MI, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const { + switch (MI.getOpcode()) { + case TargetOpcode::G_SITOFP: + case TargetOpcode::G_UITOFP: + case TargetOpcode::G_EXTRACT_VECTOR_ELT: + case TargetOpcode::G_INSERT_VECTOR_ELT: + return true; + default: + break; + } + return hasFPConstraints(MI, MRI, TRI); +} + const RegisterBankInfo::InstructionMapping & AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { const unsigned Opc = MI.getOpcode(); @@ -470,10 +537,6 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case TargetOpcode::G_AND: case TargetOpcode::G_OR: case TargetOpcode::G_XOR: - // Shifts. - case TargetOpcode::G_SHL: - case TargetOpcode::G_LSHR: - case TargetOpcode::G_ASHR: // Floating point ops. case TargetOpcode::G_FADD: case TargetOpcode::G_FSUB: @@ -488,6 +551,17 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { getFPExtMapping(DstTy.getSizeInBits(), SrcTy.getSizeInBits()), /*NumOperands*/ 2); } + // Shifts. + case TargetOpcode::G_SHL: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_ASHR: { + LLT ShiftAmtTy = MRI.getType(MI.getOperand(2).getReg()); + LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); + if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32) + return getInstructionMapping(DefaultMappingID, 1, + &ValMappings[Shift64Imm], 3); + return getSameKindOfOperandsMapping(MI); + } case TargetOpcode::COPY: { unsigned DstReg = MI.getOperand(0).getReg(); unsigned SrcReg = MI.getOperand(1).getReg(); @@ -563,10 +637,14 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { switch (Opc) { case TargetOpcode::G_SITOFP: case TargetOpcode::G_UITOFP: + if (MRI.getType(MI.getOperand(0).getReg()).isVector()) + break; OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR}; break; case TargetOpcode::G_FPTOSI: case TargetOpcode::G_FPTOUI: + if (MRI.getType(MI.getOperand(0).getReg()).isVector()) + break; OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR}; break; case TargetOpcode::G_FCMP: @@ -600,15 +678,7 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { // assume this was a floating point load in the IR. // If it was not, we would have had a bitcast before // reaching that instruction. - unsigned UseOpc = UseMI.getOpcode(); - if (isPreISelGenericFloatingPointOpcode(UseOpc) || - // Check if we feed a copy-like instruction with - // floating point constraints. In that case, we are still - // feeding fp instructions, but indirectly - // (e.g., through ABI copies). - ((UseOpc == TargetOpcode::COPY || UseMI.isPHI()) && - getRegBank(UseMI.getOperand(0).getReg(), MRI, TRI) == - &AArch64::FPRRegBank)) { + if (onlyUsesFP(UseMI, MRI, TRI)) { OpRegBankIdx[0] = PMI_FirstFPR; break; } @@ -621,18 +691,134 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { if (!VReg) break; MachineInstr *DefMI = MRI.getVRegDef(VReg); - unsigned DefOpc = DefMI->getOpcode(); - if (isPreISelGenericFloatingPointOpcode(DefOpc) || - // Check if we come from a copy-like instruction with - // floating point constraints. In that case, we are still - // fed by fp instructions, but indirectly - // (e.g., through ABI copies). - ((DefOpc == TargetOpcode::COPY || DefMI->isPHI()) && - getRegBank(DefMI->getOperand(0).getReg(), MRI, TRI) == - &AArch64::FPRRegBank)) + if (onlyDefinesFP(*DefMI, MRI, TRI)) OpRegBankIdx[0] = PMI_FirstFPR; break; } + break; + case TargetOpcode::G_SELECT: { + // If the destination is FPR, preserve that. + if (OpRegBankIdx[0] != PMI_FirstGPR) + break; + + // If we're taking in vectors, we have no choice but to put everything on + // FPRs. + LLT SrcTy = MRI.getType(MI.getOperand(2).getReg()); + if (SrcTy.isVector()) { + for (unsigned Idx = 0; Idx < 4; ++Idx) + OpRegBankIdx[Idx] = PMI_FirstFPR; + break; + } + + // Try to minimize the number of copies. If we have more floating point + // constrained values than not, then we'll put everything on FPR. Otherwise, + // everything has to be on GPR. + unsigned NumFP = 0; + + // Check if the uses of the result always produce floating point values. + // + // For example: + // + // %z = G_SELECT %cond %x %y + // fpr = G_FOO %z ... + if (any_of( + MRI.use_instructions(MI.getOperand(0).getReg()), + [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) + ++NumFP; + + // Check if the defs of the source values always produce floating point + // values. + // + // For example: + // + // %x = G_SOMETHING_ALWAYS_FLOAT %a ... + // %z = G_SELECT %cond %x %y + // + // Also check whether or not the sources have already been decided to be + // FPR. Keep track of this. + // + // This doesn't check the condition, since it's just whatever is in NZCV. + // This isn't passed explicitly in a register to fcsel/csel. + for (unsigned Idx = 2; Idx < 4; ++Idx) { + unsigned VReg = MI.getOperand(Idx).getReg(); + MachineInstr *DefMI = MRI.getVRegDef(VReg); + if (getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank || + onlyDefinesFP(*DefMI, MRI, TRI)) + ++NumFP; + } + + // If we have more FP constraints than not, then move everything over to + // FPR. + if (NumFP >= 2) + for (unsigned Idx = 0; Idx < 4; ++Idx) + OpRegBankIdx[Idx] = PMI_FirstFPR; + + break; + } + case TargetOpcode::G_UNMERGE_VALUES: { + // If the first operand belongs to a FPR register bank, then make sure that + // we preserve that. + if (OpRegBankIdx[0] != PMI_FirstGPR) + break; + + LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg()); + // UNMERGE into scalars from a vector should always use FPR. + // Likewise if any of the uses are FP instructions. + if (SrcTy.isVector() || + any_of(MRI.use_instructions(MI.getOperand(0).getReg()), + [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) { + // Set the register bank of every operand to FPR. + for (unsigned Idx = 0, NumOperands = MI.getNumOperands(); + Idx < NumOperands; ++Idx) + OpRegBankIdx[Idx] = PMI_FirstFPR; + } + break; + } + case TargetOpcode::G_EXTRACT_VECTOR_ELT: + // Destination and source need to be FPRs. + OpRegBankIdx[0] = PMI_FirstFPR; + OpRegBankIdx[1] = PMI_FirstFPR; + + // Index needs to be a GPR. + OpRegBankIdx[2] = PMI_FirstGPR; + break; + case TargetOpcode::G_INSERT_VECTOR_ELT: + OpRegBankIdx[0] = PMI_FirstFPR; + OpRegBankIdx[1] = PMI_FirstFPR; + + // The element may be either a GPR or FPR. Preserve that behaviour. + if (getRegBank(MI.getOperand(2).getReg(), MRI, TRI) == &AArch64::FPRRegBank) + OpRegBankIdx[2] = PMI_FirstFPR; + else + OpRegBankIdx[2] = PMI_FirstGPR; + + // Index needs to be a GPR. + OpRegBankIdx[3] = PMI_FirstGPR; + break; + case TargetOpcode::G_BUILD_VECTOR: + // If the first source operand belongs to a FPR register bank, then make + // sure that we preserve that. + if (OpRegBankIdx[1] != PMI_FirstGPR) + break; + unsigned VReg = MI.getOperand(1).getReg(); + if (!VReg) + break; + + // Get the instruction that defined the source operand reg, and check if + // it's a floating point operation. Or, if it's a type like s16 which + // doesn't have a exact size gpr register class. + MachineInstr *DefMI = MRI.getVRegDef(VReg); + unsigned DefOpc = DefMI->getOpcode(); + const LLT SrcTy = MRI.getType(VReg); + if (isPreISelGenericFloatingPointOpcode(DefOpc) || + SrcTy.getSizeInBits() < 32) { + // Have a floating point op. + // Make sure every operand gets mapped to a FPR register class. + unsigned NumOperands = MI.getNumOperands(); + for (unsigned Idx = 0; Idx < NumOperands; ++Idx) + OpRegBankIdx[Idx] = PMI_FirstFPR; + } + break; } // Finally construct the computed mapping. diff --git a/lib/Target/AArch64/AArch64RegisterBankInfo.h b/lib/Target/AArch64/AArch64RegisterBankInfo.h index 008221dbef58..016fed65eb2a 100644 --- a/lib/Target/AArch64/AArch64RegisterBankInfo.h +++ b/lib/Target/AArch64/AArch64RegisterBankInfo.h @@ -1,9 +1,8 @@ //===- AArch64RegisterBankInfo -----------------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -58,6 +57,7 @@ protected: FPExt16To64Idx = 43, FPExt32To64Idx = 45, FPExt64To128Idx = 47, + Shift64Imm = 49 }; static bool checkPartialMap(unsigned Idx, unsigned ValStartIdx, @@ -114,6 +114,18 @@ class AArch64RegisterBankInfo final : public AArch64GenRegisterBankInfo { const InstructionMapping & getSameKindOfOperandsMapping(const MachineInstr &MI) const; + /// Returns true if the output of \p MI must be stored on a FPR register. + bool hasFPConstraints(const MachineInstr &MI, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const; + + /// Returns true if the source registers of \p MI must all be FPRs. + bool onlyUsesFP(const MachineInstr &MI, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const; + + /// Returns true if the destination register of \p MI must be a FPR. + bool onlyDefinesFP(const MachineInstr &MI, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const; + public: AArch64RegisterBankInfo(const TargetRegisterInfo &TRI); diff --git a/lib/Target/AArch64/AArch64RegisterBanks.td b/lib/Target/AArch64/AArch64RegisterBanks.td index eee584708f69..7bbd992890d1 100644 --- a/lib/Target/AArch64/AArch64RegisterBanks.td +++ b/lib/Target/AArch64/AArch64RegisterBanks.td @@ -1,9 +1,8 @@ //=- AArch64RegisterBank.td - Describe the AArch64 Banks -----*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64RegisterInfo.cpp b/lib/Target/AArch64/AArch64RegisterInfo.cpp index 96ae45ae3d0d..6d5a4e3d2f76 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.cpp +++ b/lib/Target/AArch64/AArch64RegisterInfo.cpp @@ -1,9 +1,8 @@ //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -217,11 +216,8 @@ bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF, } bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const { - // FIXME: Get the list of argument registers from TableGen. - static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2, - AArch64::X3, AArch64::X4, AArch64::X5, - AArch64::X6, AArch64::X7 }; - return std::any_of(std::begin(GPRArgRegs), std::end(GPRArgRegs), + return std::any_of(std::begin(*AArch64::GPR64argRegClass.MC), + std::end(*AArch64::GPR64argRegClass.MC), [this, &MF](MCPhysReg r){return isReservedReg(MF, r);}); } @@ -283,7 +279,7 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { return false; } -unsigned +Register AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const { const AArch64FrameLowering *TFI = getFrameLowering(MF); return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP; @@ -457,15 +453,34 @@ void AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, if (MI.isDebugValue() || MI.getOpcode() == TargetOpcode::STACKMAP || MI.getOpcode() == TargetOpcode::PATCHPOINT) { Offset = TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg, - /*PreferFP=*/true); + /*PreferFP=*/true, + /*ForSimm=*/false); Offset += MI.getOperand(FIOperandNum + 1).getImm(); MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false /*isDef*/); MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset); return; } + if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) { + MachineOperand &FI = MI.getOperand(FIOperandNum); + Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex); + FI.ChangeToImmediate(Offset); + return; + } + + if (MI.getOpcode() == AArch64::TAGPstack) { + // TAGPstack must use the virtual frame register in its 3rd operand. + const MachineFrameInfo &MFI = MF.getFrameInfo(); + const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); + FrameReg = MI.getOperand(3).getReg(); + Offset = + MFI.getObjectOffset(FrameIndex) + AFI->getTaggedBasePointerOffset(); + } else { + Offset = TFI->resolveFrameIndexReference( + MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true); + } + // Modify MI as necessary to handle as much of 'Offset' as possible - Offset = TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg); if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII)) return; @@ -519,3 +534,13 @@ unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, return 16; } } + +unsigned AArch64RegisterInfo::getLocalAddressRegister( + const MachineFunction &MF) const { + const auto &MFI = MF.getFrameInfo(); + if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects()) + return AArch64::SP; + else if (needsStackRealignment(MF)) + return getBaseRegister(); + return getFrameRegister(MF); +} diff --git a/lib/Target/AArch64/AArch64RegisterInfo.h b/lib/Target/AArch64/AArch64RegisterInfo.h index c4153228a7c0..2c3f82c530d8 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.h +++ b/lib/Target/AArch64/AArch64RegisterInfo.h @@ -1,9 +1,8 @@ //==- AArch64RegisterInfo.h - AArch64 Register Information Impl --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -114,7 +113,7 @@ public: unsigned getBaseRegister() const; // Debug information queries. - unsigned getFrameRegister(const MachineFunction &MF) const override; + Register getFrameRegister(const MachineFunction &MF) const override; unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const override; @@ -122,6 +121,8 @@ public: bool trackLivenessAfterRegAlloc(const MachineFunction&) const override { return true; } + + unsigned getLocalAddressRegister(const MachineFunction &MF) const; }; } // end namespace llvm diff --git a/lib/Target/AArch64/AArch64RegisterInfo.td b/lib/Target/AArch64/AArch64RegisterInfo.td index d3710cea0687..61fc0795c242 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.td +++ b/lib/Target/AArch64/AArch64RegisterInfo.td @@ -1,9 +1,8 @@ //=- AArch64RegisterInfo.td - Describe the AArch64 Registers -*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -188,6 +187,10 @@ def GPR64z : RegisterOperand<GPR64> { let GIZeroRegister = XZR; } +// GPR argument registers. +def GPR32arg : RegisterClass<"AArch64", [i32], 32, (sequence "W%u", 0, 7)>; +def GPR64arg : RegisterClass<"AArch64", [i64], 64, (sequence "X%u", 0, 7)>; + // GPR register classes which include WZR/XZR AND SP/WSP. This is not a // constraint used by any instructions, it is used as a common super-class. def GPR32all : RegisterClass<"AArch64", [i32], 32, (add GPR32common, WZR, WSP)>; @@ -206,6 +209,11 @@ def tcGPR64 : RegisterClass<"AArch64", [i64], 64, (sub GPR64common, X19, X20, X2 // BTI-protected function. def rtcGPR64 : RegisterClass<"AArch64", [i64], 64, (add X16, X17)>; +// Register set that excludes registers that are reserved for procedure calls. +// This is used for pseudo-instructions that are actually implemented using a +// procedure call. +def GPR64noip : RegisterClass<"AArch64", [i64], 64, (sub GPR64, X16, X17, LR)>; + // GPR register classes for post increment amount of vector load/store that // has alternate printing when Rm=31 and prints a constant immediate value // equal to the total number of bytes transferred. @@ -649,10 +657,12 @@ def FPR128Op : RegisterOperand<FPR128, "printOperand"> { // ARMv8.1a atomic CASP register operands -def WSeqPairs : RegisterTuples<[sube32, subo32], - [(rotl GPR32, 0), (rotl GPR32, 1)]>; -def XSeqPairs : RegisterTuples<[sube64, subo64], - [(rotl GPR64, 0), (rotl GPR64, 1)]>; +def WSeqPairs : RegisterTuples<[sube32, subo32], + [(decimate (rotl GPR32, 0), 2), + (decimate (rotl GPR32, 1), 2)]>; +def XSeqPairs : RegisterTuples<[sube64, subo64], + [(decimate (rotl GPR64, 0), 2), + (decimate (rotl GPR64, 1), 2)]>; def WSeqPairsClass : RegisterClass<"AArch64", [untyped], 32, (add WSeqPairs)>{ diff --git a/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp b/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp index af555f6d2266..854670079e40 100644 --- a/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp +++ b/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp @@ -1,8 +1,7 @@ // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64SVEInstrInfo.td b/lib/Target/AArch64/AArch64SVEInstrInfo.td index 0fde68011e86..79ab42f4c080 100644 --- a/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1,9 +1,8 @@ //=- AArch64SVEInstrInfo.td - AArch64 SVE Instructions -*- tablegen -*-----=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,10 +25,10 @@ let Predicates = [HasSVE] in { defm SQSUB_ZZZ : sve_int_bin_cons_arit_0<0b110, "sqsub">; defm UQSUB_ZZZ : sve_int_bin_cons_arit_0<0b111, "uqsub">; - def AND_ZZZ : sve_int_bin_cons_log<0b00, "and">; - def ORR_ZZZ : sve_int_bin_cons_log<0b01, "orr">; - def EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor">; - def BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic">; + defm AND_ZZZ : sve_int_bin_cons_log<0b00, "and">; + defm ORR_ZZZ : sve_int_bin_cons_log<0b01, "orr">; + defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor">; + defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic">; defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add">; defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub">; @@ -876,10 +875,10 @@ let Predicates = [HasSVE] in { defm LSL_WIDE_ZZZ : sve_int_bin_cons_shift_wide<0b11, "lsl">; // Predicated shifts - defm ASR_ZPmI : sve_int_bin_pred_shift_imm_right<0b000, "asr">; - defm LSR_ZPmI : sve_int_bin_pred_shift_imm_right<0b001, "lsr">; - defm LSL_ZPmI : sve_int_bin_pred_shift_imm_left< 0b011, "lsl">; - defm ASRD_ZPmI : sve_int_bin_pred_shift_imm_right<0b100, "asrd">; + defm ASR_ZPmI : sve_int_bin_pred_shift_imm_right<0b0000, "asr">; + defm LSR_ZPmI : sve_int_bin_pred_shift_imm_right<0b0001, "lsr">; + defm LSL_ZPmI : sve_int_bin_pred_shift_imm_left< 0b0011, "lsl">; + defm ASRD_ZPmI : sve_int_bin_pred_shift_imm_right<0b0100, "asrd">; defm ASR_ZPmZ : sve_int_bin_pred_shift<0b000, "asr">; defm LSR_ZPmZ : sve_int_bin_pred_shift<0b001, "lsr">; @@ -1022,3 +1021,406 @@ let Predicates = [HasSVE] in { def : InstAlias<"fcmlt $Zd, $Pg/z, $Zm, $Zn", (FCMGT_PPzZZ_D PPR64:$Zd, PPR3bAny:$Pg, ZPR64:$Zn, ZPR64:$Zm), 0>; } + +let Predicates = [HasSVE2] in { + // SVE2 integer multiply-add (indexed) + defm MLA_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b0, "mla">; + defm MLS_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b1, "mls">; + + // SVE2 saturating multiply-add high (indexed) + defm SQRDMLAH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b0, "sqrdmlah">; + defm SQRDMLSH_ZZZI : sve2_int_mla_by_indexed_elem<0b10, 0b1, "sqrdmlsh">; + + // SVE2 saturating multiply-add high (vectors, unpredicated) + defm SQRDMLAH_ZZZ : sve2_int_mla<0b0, "sqrdmlah">; + defm SQRDMLSH_ZZZ : sve2_int_mla<0b1, "sqrdmlsh">; + + // SVE2 integer multiply (indexed) + defm MUL_ZZZI : sve2_int_mul_by_indexed_elem<0b1110, "mul">; + + // SVE2 saturating multiply high (indexed) + defm SQDMULH_ZZZI : sve2_int_mul_by_indexed_elem<0b1100, "sqdmulh">; + defm SQRDMULH_ZZZI : sve2_int_mul_by_indexed_elem<0b1101, "sqrdmulh">; + + // SVE2 signed saturating doubling multiply high (unpredicated) + defm SQDMULH_ZZZ : sve2_int_mul<0b100, "sqdmulh">; + defm SQRDMULH_ZZZ : sve2_int_mul<0b101, "sqrdmulh">; + + // SVE2 integer multiply vectors (unpredicated) + defm MUL_ZZZ : sve2_int_mul<0b000, "mul">; + defm SMULH_ZZZ : sve2_int_mul<0b010, "smulh">; + defm UMULH_ZZZ : sve2_int_mul<0b011, "umulh">; + def PMUL_ZZZ_B : sve2_int_mul<0b00, 0b001, "pmul", ZPR8>; + + // SVE2 complex integer dot product (indexed) + defm CDOT_ZZZI : sve2_cintx_dot_by_indexed_elem<"cdot">; + + // SVE2 complex integer dot product + defm CDOT_ZZZ : sve2_cintx_dot<"cdot">; + + // SVE2 complex integer multiply-add (indexed) + defm CMLA_ZZZI : sve2_cmla_by_indexed_elem<0b0, "cmla">; + // SVE2 complex saturating multiply-add (indexed) + defm SQRDCMLAH_ZZZI : sve2_cmla_by_indexed_elem<0b1, "sqrdcmlah">; + + // SVE2 complex integer multiply-add + defm CMLA_ZZZ : sve2_int_cmla<0b0, "cmla">; + defm SQRDCMLAH_ZZZ : sve2_int_cmla<0b1, "sqrdcmlah">; + + // SVE2 integer multiply long (indexed) + defm SMULLB_ZZZI : sve2_int_mul_long_by_indexed_elem<0b000, "smullb">; + defm SMULLT_ZZZI : sve2_int_mul_long_by_indexed_elem<0b001, "smullt">; + defm UMULLB_ZZZI : sve2_int_mul_long_by_indexed_elem<0b010, "umullb">; + defm UMULLT_ZZZI : sve2_int_mul_long_by_indexed_elem<0b011, "umullt">; + + // SVE2 saturating multiply (indexed) + defm SQDMULLB_ZZZI : sve2_int_mul_long_by_indexed_elem<0b100, "sqdmullb">; + defm SQDMULLT_ZZZI : sve2_int_mul_long_by_indexed_elem<0b101, "sqdmullt">; + + // SVE2 integer multiply-add long (indexed) + defm SMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1000, "smlalb">; + defm SMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1001, "smlalt">; + defm UMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1010, "umlalb">; + defm UMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1011, "umlalt">; + defm SMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1100, "smlslb">; + defm SMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1101, "smlslt">; + defm UMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1110, "umlslb">; + defm UMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1111, "umlslt">; + + // SVE2 integer multiply-add long (vectors, unpredicated) + defm SMLALB_ZZZ : sve2_int_mla_long<0b10000, "smlalb">; + defm SMLALT_ZZZ : sve2_int_mla_long<0b10001, "smlalt">; + defm UMLALB_ZZZ : sve2_int_mla_long<0b10010, "umlalb">; + defm UMLALT_ZZZ : sve2_int_mla_long<0b10011, "umlalt">; + defm SMLSLB_ZZZ : sve2_int_mla_long<0b10100, "smlslb">; + defm SMLSLT_ZZZ : sve2_int_mla_long<0b10101, "smlslt">; + defm UMLSLB_ZZZ : sve2_int_mla_long<0b10110, "umlslb">; + defm UMLSLT_ZZZ : sve2_int_mla_long<0b10111, "umlslt">; + + // SVE2 saturating multiply-add long (indexed) + defm SQDMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0100, "sqdmlalb">; + defm SQDMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0101, "sqdmlalt">; + defm SQDMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0110, "sqdmlslb">; + defm SQDMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0111, "sqdmlslt">; + + // SVE2 saturating multiply-add long (vectors, unpredicated) + defm SQDMLALB_ZZZ : sve2_int_mla_long<0b11000, "sqdmlalb">; + defm SQDMLALT_ZZZ : sve2_int_mla_long<0b11001, "sqdmlalt">; + defm SQDMLSLB_ZZZ : sve2_int_mla_long<0b11010, "sqdmlslb">; + defm SQDMLSLT_ZZZ : sve2_int_mla_long<0b11011, "sqdmlslt">; + + // SVE2 saturating multiply-add interleaved long + defm SQDMLALBT_ZZZ : sve2_int_mla_long<0b00010, "sqdmlalbt">; + defm SQDMLSLBT_ZZZ : sve2_int_mla_long<0b00011, "sqdmlslbt">; + + // SVE2 integer halving add/subtract (predicated) + defm SHADD_ZPmZ : sve2_int_arith_pred<0b100000, "shadd">; + defm UHADD_ZPmZ : sve2_int_arith_pred<0b100010, "uhadd">; + defm SHSUB_ZPmZ : sve2_int_arith_pred<0b100100, "shsub">; + defm UHSUB_ZPmZ : sve2_int_arith_pred<0b100110, "uhsub">; + defm SRHADD_ZPmZ : sve2_int_arith_pred<0b101000, "srhadd">; + defm URHADD_ZPmZ : sve2_int_arith_pred<0b101010, "urhadd">; + defm SHSUBR_ZPmZ : sve2_int_arith_pred<0b101100, "shsubr">; + defm UHSUBR_ZPmZ : sve2_int_arith_pred<0b101110, "uhsubr">; + + // SVE2 integer pairwise add and accumulate long + defm SADALP_ZPmZ : sve2_int_sadd_long_accum_pairwise<0, "sadalp">; + defm UADALP_ZPmZ : sve2_int_sadd_long_accum_pairwise<1, "uadalp">; + + // SVE2 integer pairwise arithmetic + defm ADDP_ZPmZ : sve2_int_arith_pred<0b100011, "addp">; + defm SMAXP_ZPmZ : sve2_int_arith_pred<0b101001, "smaxp">; + defm UMAXP_ZPmZ : sve2_int_arith_pred<0b101011, "umaxp">; + defm SMINP_ZPmZ : sve2_int_arith_pred<0b101101, "sminp">; + defm UMINP_ZPmZ : sve2_int_arith_pred<0b101111, "uminp">; + + // SVE2 integer unary operations (predicated) + defm URECPE_ZPmZ : sve2_int_un_pred_arit_s<0b000, "urecpe">; + defm URSQRTE_ZPmZ : sve2_int_un_pred_arit_s<0b001, "ursqrte">; + defm SQABS_ZPmZ : sve2_int_un_pred_arit<0b100, "sqabs">; + defm SQNEG_ZPmZ : sve2_int_un_pred_arit<0b101, "sqneg">; + + // SVE2 saturating add/subtract + defm SQADD_ZPmZ : sve2_int_arith_pred<0b110000, "sqadd">; + defm UQADD_ZPmZ : sve2_int_arith_pred<0b110010, "uqadd">; + defm SQSUB_ZPmZ : sve2_int_arith_pred<0b110100, "sqsub">; + defm UQSUB_ZPmZ : sve2_int_arith_pred<0b110110, "uqsub">; + defm SUQADD_ZPmZ : sve2_int_arith_pred<0b111000, "suqadd">; + defm USQADD_ZPmZ : sve2_int_arith_pred<0b111010, "usqadd">; + defm SQSUBR_ZPmZ : sve2_int_arith_pred<0b111100, "sqsubr">; + defm UQSUBR_ZPmZ : sve2_int_arith_pred<0b111110, "uqsubr">; + + // SVE2 saturating/rounding bitwise shift left (predicated) + defm SRSHL_ZPmZ : sve2_int_arith_pred<0b000100, "srshl">; + defm URSHL_ZPmZ : sve2_int_arith_pred<0b000110, "urshl">; + defm SRSHLR_ZPmZ : sve2_int_arith_pred<0b001100, "srshlr">; + defm URSHLR_ZPmZ : sve2_int_arith_pred<0b001110, "urshlr">; + defm SQSHL_ZPmZ : sve2_int_arith_pred<0b010000, "sqshl">; + defm UQSHL_ZPmZ : sve2_int_arith_pred<0b010010, "uqshl">; + defm SQRSHL_ZPmZ : sve2_int_arith_pred<0b010100, "sqrshl">; + defm UQRSHL_ZPmZ : sve2_int_arith_pred<0b010110, "uqrshl">; + defm SQSHLR_ZPmZ : sve2_int_arith_pred<0b011000, "sqshlr">; + defm UQSHLR_ZPmZ : sve2_int_arith_pred<0b011010, "uqshlr">; + defm SQRSHLR_ZPmZ : sve2_int_arith_pred<0b011100, "sqrshlr">; + defm UQRSHLR_ZPmZ : sve2_int_arith_pred<0b011110, "uqrshlr">; + + // SVE2 integer add/subtract long + defm SADDLB_ZZZ : sve2_wide_int_arith_long<0b00000, "saddlb">; + defm SADDLT_ZZZ : sve2_wide_int_arith_long<0b00001, "saddlt">; + defm UADDLB_ZZZ : sve2_wide_int_arith_long<0b00010, "uaddlb">; + defm UADDLT_ZZZ : sve2_wide_int_arith_long<0b00011, "uaddlt">; + defm SSUBLB_ZZZ : sve2_wide_int_arith_long<0b00100, "ssublb">; + defm SSUBLT_ZZZ : sve2_wide_int_arith_long<0b00101, "ssublt">; + defm USUBLB_ZZZ : sve2_wide_int_arith_long<0b00110, "usublb">; + defm USUBLT_ZZZ : sve2_wide_int_arith_long<0b00111, "usublt">; + defm SABDLB_ZZZ : sve2_wide_int_arith_long<0b01100, "sabdlb">; + defm SABDLT_ZZZ : sve2_wide_int_arith_long<0b01101, "sabdlt">; + defm UABDLB_ZZZ : sve2_wide_int_arith_long<0b01110, "uabdlb">; + defm UABDLT_ZZZ : sve2_wide_int_arith_long<0b01111, "uabdlt">; + + // SVE2 integer add/subtract wide + defm SADDWB_ZZZ : sve2_wide_int_arith_wide<0b000, "saddwb">; + defm SADDWT_ZZZ : sve2_wide_int_arith_wide<0b001, "saddwt">; + defm UADDWB_ZZZ : sve2_wide_int_arith_wide<0b010, "uaddwb">; + defm UADDWT_ZZZ : sve2_wide_int_arith_wide<0b011, "uaddwt">; + defm SSUBWB_ZZZ : sve2_wide_int_arith_wide<0b100, "ssubwb">; + defm SSUBWT_ZZZ : sve2_wide_int_arith_wide<0b101, "ssubwt">; + defm USUBWB_ZZZ : sve2_wide_int_arith_wide<0b110, "usubwb">; + defm USUBWT_ZZZ : sve2_wide_int_arith_wide<0b111, "usubwt">; + + // SVE2 integer multiply long + defm SQDMULLB_ZZZ : sve2_wide_int_arith_long<0b11000, "sqdmullb">; + defm SQDMULLT_ZZZ : sve2_wide_int_arith_long<0b11001, "sqdmullt">; + defm SMULLB_ZZZ : sve2_wide_int_arith_long<0b11100, "smullb">; + defm SMULLT_ZZZ : sve2_wide_int_arith_long<0b11101, "smullt">; + defm UMULLB_ZZZ : sve2_wide_int_arith_long<0b11110, "umullb">; + defm UMULLT_ZZZ : sve2_wide_int_arith_long<0b11111, "umullt">; + defm PMULLB_ZZZ : sve2_pmul_long<0b0, "pmullb">; + defm PMULLT_ZZZ : sve2_pmul_long<0b1, "pmullt">; + + // SVE2 bitwise shift and insert + defm SRI_ZZI : sve2_int_bin_cons_shift_imm_right<0b0, "sri">; + defm SLI_ZZI : sve2_int_bin_cons_shift_imm_left< 0b1, "sli">; + + // SVE2 bitwise shift right and accumulate + defm SSRA_ZZI : sve2_int_bin_accum_cons_shift_imm_right<0b00, "ssra">; + defm USRA_ZZI : sve2_int_bin_accum_cons_shift_imm_right<0b01, "usra">; + defm SRSRA_ZZI : sve2_int_bin_accum_cons_shift_imm_right<0b10, "srsra">; + defm URSRA_ZZI : sve2_int_bin_accum_cons_shift_imm_right<0b11, "ursra">; + + // SVE2 complex integer add + defm CADD_ZZI : sve2_int_cadd<0b0, "cadd">; + defm SQCADD_ZZI : sve2_int_cadd<0b1, "sqcadd">; + + // SVE2 integer absolute difference and accumulate + defm SABA_ZZZ : sve2_int_absdiff_accum<0b0, "saba">; + defm UABA_ZZZ : sve2_int_absdiff_accum<0b1, "uaba">; + + // SVE2 integer absolute difference and accumulate long + defm SABALB_ZZZ : sve2_int_absdiff_accum_long<0b00, "sabalb">; + defm SABALT_ZZZ : sve2_int_absdiff_accum_long<0b01, "sabalt">; + defm UABALB_ZZZ : sve2_int_absdiff_accum_long<0b10, "uabalb">; + defm UABALT_ZZZ : sve2_int_absdiff_accum_long<0b11, "uabalt">; + + // SVE2 integer add/subtract long with carry + defm ADCLB_ZZZ : sve2_int_addsub_long_carry<0b00, "adclb">; + defm ADCLT_ZZZ : sve2_int_addsub_long_carry<0b01, "adclt">; + defm SBCLB_ZZZ : sve2_int_addsub_long_carry<0b10, "sbclb">; + defm SBCLT_ZZZ : sve2_int_addsub_long_carry<0b11, "sbclt">; + + // SVE2 bitwise shift right narrow + defm SQSHRUNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0000, "sqshrunb">; + defm SQSHRUNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0001, "sqshrunt">; + defm SQRSHRUNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0010, "sqrshrunb">; + defm SQRSHRUNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0011, "sqrshrunt">; + defm SHRNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0100, "shrnb">; + defm SHRNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0101, "shrnt">; + defm RSHRNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0110, "rshrnb">; + defm RSHRNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b0111, "rshrnt">; + defm SQSHRNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1000, "sqshrnb">; + defm SQSHRNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1001, "sqshrnt">; + defm SQRSHRNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1010, "sqrshrnb">; + defm SQRSHRNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1011, "sqrshrnt">; + defm UQSHRNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1100, "uqshrnb">; + defm UQSHRNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1101, "uqshrnt">; + defm UQRSHRNB_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1110, "uqrshrnb">; + defm UQRSHRNT_ZZI : sve2_int_bin_cons_shift_imm_right_narrow<0b1111, "uqrshrnt">; + + // SVE2 integer add/subtract narrow high part + defm ADDHNB_ZZZ : sve2_int_addsub_narrow_high<0b000, "addhnb">; + defm ADDHNT_ZZZ : sve2_int_addsub_narrow_high<0b001, "addhnt">; + defm RADDHNB_ZZZ : sve2_int_addsub_narrow_high<0b010, "raddhnb">; + defm RADDHNT_ZZZ : sve2_int_addsub_narrow_high<0b011, "raddhnt">; + defm SUBHNB_ZZZ : sve2_int_addsub_narrow_high<0b100, "subhnb">; + defm SUBHNT_ZZZ : sve2_int_addsub_narrow_high<0b101, "subhnt">; + defm RSUBHNB_ZZZ : sve2_int_addsub_narrow_high<0b110, "rsubhnb">; + defm RSUBHNT_ZZZ : sve2_int_addsub_narrow_high<0b111, "rsubhnt">; + + // SVE2 saturating extract narrow + defm SQXTNB_ZZ : sve2_int_sat_extract_narrow<0b000, "sqxtnb">; + defm SQXTNT_ZZ : sve2_int_sat_extract_narrow<0b001, "sqxtnt">; + defm UQXTNB_ZZ : sve2_int_sat_extract_narrow<0b010, "uqxtnb">; + defm UQXTNT_ZZ : sve2_int_sat_extract_narrow<0b011, "uqxtnt">; + defm SQXTUNB_ZZ : sve2_int_sat_extract_narrow<0b100, "sqxtunb">; + defm SQXTUNT_ZZ : sve2_int_sat_extract_narrow<0b101, "sqxtunt">; + + // SVE2 character match + defm MATCH_PPzZZ : sve2_char_match<0b0, "match">; + defm NMATCH_PPzZZ : sve2_char_match<0b1, "nmatch">; + + // SVE2 bitwise exclusive-or interleaved + defm EORBT_ZZZ : sve2_bitwise_xor_interleaved<0b0, "eorbt">; + defm EORTB_ZZZ : sve2_bitwise_xor_interleaved<0b1, "eortb">; + + // SVE2 bitwise shift left long + defm SSHLLB_ZZI : sve2_bitwise_shift_left_long<0b00, "sshllb">; + defm SSHLLT_ZZI : sve2_bitwise_shift_left_long<0b01, "sshllt">; + defm USHLLB_ZZI : sve2_bitwise_shift_left_long<0b10, "ushllb">; + defm USHLLT_ZZI : sve2_bitwise_shift_left_long<0b11, "ushllt">; + + // SVE2 integer add/subtract interleaved long + defm SADDLBT_ZZZ : sve2_misc_int_addsub_long_interleaved<0b00, "saddlbt">; + defm SSUBLBT_ZZZ : sve2_misc_int_addsub_long_interleaved<0b10, "ssublbt">; + defm SSUBLTB_ZZZ : sve2_misc_int_addsub_long_interleaved<0b11, "ssubltb">; + + // SVE2 histogram generation (segment) + def HISTSEG_ZZZ : sve2_hist_gen_segment<"histseg">; + + // SVE2 histogram generation (vector) + defm HISTCNT_ZPzZZ : sve2_hist_gen_vector<"histcnt">; + + // SVE2 floating-point convert precision + defm FCVTXNT_ZPmZ : sve2_fp_convert_down_odd_rounding<"fcvtxnt">; + defm FCVTNT_ZPmZ : sve2_fp_convert_down_narrow<"fcvtnt">; + defm FCVTLT_ZPmZ : sve2_fp_convert_up_long<"fcvtlt">; + + // SVE2 floating-point pairwise operations + defm FADDP_ZPmZZ : sve2_fp_pairwise_pred<0b000, "faddp">; + defm FMAXNMP_ZPmZZ : sve2_fp_pairwise_pred<0b100, "fmaxnmp">; + defm FMINNMP_ZPmZZ : sve2_fp_pairwise_pred<0b101, "fminnmp">; + defm FMAXP_ZPmZZ : sve2_fp_pairwise_pred<0b110, "fmaxp">; + defm FMINP_ZPmZZ : sve2_fp_pairwise_pred<0b111, "fminp">; + + // SVE2 floating-point multiply-add long (indexed) + def FMLALB_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b00, "fmlalb">; + def FMLALT_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b01, "fmlalt">; + def FMLSLB_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b10, "fmlslb">; + def FMLSLT_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b11, "fmlslt">; + + // SVE2 floating-point multiply-add long + def FMLALB_ZZZ_SHH : sve2_fp_mla_long<0b00, "fmlalb">; + def FMLALT_ZZZ_SHH : sve2_fp_mla_long<0b01, "fmlalt">; + def FMLSLB_ZZZ_SHH : sve2_fp_mla_long<0b10, "fmlslb">; + def FMLSLT_ZZZ_SHH : sve2_fp_mla_long<0b11, "fmlslt">; + + // SVE2 bitwise ternary operations + defm EOR3_ZZZZ_D : sve2_int_bitwise_ternary_op<0b000, "eor3">; + defm BCAX_ZZZZ_D : sve2_int_bitwise_ternary_op<0b010, "bcax">; + def BSL_ZZZZ_D : sve2_int_bitwise_ternary_op_d<0b001, "bsl">; + def BSL1N_ZZZZ_D : sve2_int_bitwise_ternary_op_d<0b011, "bsl1n">; + def BSL2N_ZZZZ_D : sve2_int_bitwise_ternary_op_d<0b101, "bsl2n">; + def NBSL_ZZZZ_D : sve2_int_bitwise_ternary_op_d<0b111, "nbsl">; + + // sve_int_rotate_imm + defm XAR_ZZZI : sve2_int_rotate_right_imm<"xar">; + + // SVE2 extract vector (immediate offset, constructive) + def EXT_ZZI_B : sve2_int_perm_extract_i_cons<"ext">; + + // SVE floating-point convert precision + def FCVTX_ZPmZ_DtoS : sve_fp_2op_p_zd<0b0001010, "fcvtx", ZPR64, ZPR32, ElementSizeD>; + + // SVE floating-point convert to integer + defm FLOGB_ZPmZ : sve2_fp_flogb<"flogb">; + + // Non-temporal contiguous loads (vector + register) + defm LDNT1SB_ZZR_S : sve2_mem_cldnt_vs<0b00000, "ldnt1sb", Z_s, ZPR32>; + defm LDNT1B_ZZR_S : sve2_mem_cldnt_vs<0b00001, "ldnt1b", Z_s, ZPR32>; + defm LDNT1SH_ZZR_S : sve2_mem_cldnt_vs<0b00100, "ldnt1sh", Z_s, ZPR32>; + defm LDNT1H_ZZR_S : sve2_mem_cldnt_vs<0b00101, "ldnt1h", Z_s, ZPR32>; + defm LDNT1W_ZZR_S : sve2_mem_cldnt_vs<0b01001, "ldnt1w", Z_s, ZPR32>; + + defm LDNT1SB_ZZR_D : sve2_mem_cldnt_vs<0b10000, "ldnt1sb", Z_d, ZPR64>; + defm LDNT1B_ZZR_D : sve2_mem_cldnt_vs<0b10010, "ldnt1b", Z_d, ZPR64>; + defm LDNT1SH_ZZR_D : sve2_mem_cldnt_vs<0b10100, "ldnt1sh", Z_d, ZPR64>; + defm LDNT1H_ZZR_D : sve2_mem_cldnt_vs<0b10110, "ldnt1h", Z_d, ZPR64>; + defm LDNT1SW_ZZR_D : sve2_mem_cldnt_vs<0b11000, "ldnt1sw", Z_d, ZPR64>; + defm LDNT1W_ZZR_D : sve2_mem_cldnt_vs<0b11010, "ldnt1w", Z_d, ZPR64>; + defm LDNT1D_ZZR_D : sve2_mem_cldnt_vs<0b11110, "ldnt1d", Z_d, ZPR64>; + + // SVE2 vector splice (constructive) + defm SPLICE_ZPZZ : sve2_int_perm_splice_cons<"splice">; + + // Predicated shifts + defm SQSHL_ZPmI : sve_int_bin_pred_shift_imm_left< 0b0110, "sqshl">; + defm UQSHL_ZPmI : sve_int_bin_pred_shift_imm_left< 0b0111, "uqshl">; + defm SRSHR_ZPmI : sve_int_bin_pred_shift_imm_right<0b1100, "srshr">; + defm URSHR_ZPmI : sve_int_bin_pred_shift_imm_right<0b1101, "urshr">; + defm SQSHLU_ZPmI : sve_int_bin_pred_shift_imm_left< 0b1111, "sqshlu">; + + // Non-temporal contiguous stores (vector + register) + defm STNT1B_ZZR_S : sve2_mem_cstnt_vs<0b001, "stnt1b", Z_s, ZPR32>; + defm STNT1H_ZZR_S : sve2_mem_cstnt_vs<0b011, "stnt1h", Z_s, ZPR32>; + defm STNT1W_ZZR_S : sve2_mem_cstnt_vs<0b101, "stnt1w", Z_s, ZPR32>; + + defm STNT1B_ZZR_D : sve2_mem_cstnt_vs<0b000, "stnt1b", Z_d, ZPR64>; + defm STNT1H_ZZR_D : sve2_mem_cstnt_vs<0b010, "stnt1h", Z_d, ZPR64>; + defm STNT1W_ZZR_D : sve2_mem_cstnt_vs<0b100, "stnt1w", Z_d, ZPR64>; + defm STNT1D_ZZR_D : sve2_mem_cstnt_vs<0b110, "stnt1d", Z_d, ZPR64>; + + // SVE table lookup (three sources) + defm TBL_ZZZZ : sve2_int_perm_tbl<"tbl">; + defm TBX_ZZZ : sve2_int_perm_tbx<"tbx">; + + // SVE integer compare scalar count and limit + defm WHILEGE_PWW : sve_int_while4_rr<0b000, "whilege">; + defm WHILEGT_PWW : sve_int_while4_rr<0b001, "whilegt">; + defm WHILEHS_PWW : sve_int_while4_rr<0b100, "whilehs">; + defm WHILEHI_PWW : sve_int_while4_rr<0b101, "whilehi">; + + defm WHILEGE_PXX : sve_int_while8_rr<0b000, "whilege">; + defm WHILEGT_PXX : sve_int_while8_rr<0b001, "whilegt">; + defm WHILEHS_PXX : sve_int_while8_rr<0b100, "whilehs">; + defm WHILEHI_PXX : sve_int_while8_rr<0b101, "whilehi">; + + // SVE pointer conflict compare + defm WHILEWR_PXX : sve2_int_while_rr<0b0, "whilewr">; + defm WHILERW_PXX : sve2_int_while_rr<0b1, "whilerw">; +} + +let Predicates = [HasSVE2AES] in { + // SVE2 crypto destructive binary operations + def AESE_ZZZ_B : sve2_crypto_des_bin_op<0b00, "aese", ZPR8>; + def AESD_ZZZ_B : sve2_crypto_des_bin_op<0b01, "aesd", ZPR8>; + + // SVE2 crypto unary operations + def AESMC_ZZ_B : sve2_crypto_unary_op<0b0, "aesmc">; + def AESIMC_ZZ_B : sve2_crypto_unary_op<0b1, "aesimc">; + + // PMULLB and PMULLT instructions which operate with 64-bit source and + // 128-bit destination elements are enabled with crypto extensions, similar + // to NEON PMULL2 instruction. + def PMULLB_ZZZ_Q : sve2_wide_int_arith<0b00, 0b11010, "pmullb", + ZPR128, ZPR64, ZPR64>; + def PMULLT_ZZZ_Q : sve2_wide_int_arith<0b00, 0b11011, "pmullt", + ZPR128, ZPR64, ZPR64>; +} + +let Predicates = [HasSVE2SM4] in { + // SVE2 crypto constructive binary operations + def SM4EKEY_ZZZ_S : sve2_crypto_cons_bin_op<0b0, "sm4ekey", ZPR32>; + // SVE2 crypto destructive binary operations + def SM4E_ZZZ_S : sve2_crypto_des_bin_op<0b10, "sm4e", ZPR32>; +} + +let Predicates = [HasSVE2SHA3] in { + // SVE2 crypto constructive binary operations + def RAX1_ZZZ_D : sve2_crypto_cons_bin_op<0b1, "rax1", ZPR64>; +} + +let Predicates = [HasSVE2BitPerm] in { + // SVE2 bitwise permute + defm BEXT_ZZZ : sve2_misc_bitwise<0b1100, "bext">; + defm BDEP_ZZZ : sve2_misc_bitwise<0b1101, "bdep">; + defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp">; +} diff --git a/lib/Target/AArch64/AArch64SchedA53.td b/lib/Target/AArch64/AArch64SchedA53.td index f253a4f3e25a..a6df0f3f083c 100644 --- a/lib/Target/AArch64/AArch64SchedA53.td +++ b/lib/Target/AArch64/AArch64SchedA53.td @@ -1,9 +1,8 @@ //==- AArch64SchedA53.td - Cortex-A53 Scheduling Definitions -*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,7 +26,7 @@ def CortexA53Model : SchedMachineModel { // v 1.0 Spreadsheet let CompleteModel = 1; - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; } diff --git a/lib/Target/AArch64/AArch64SchedA57.td b/lib/Target/AArch64/AArch64SchedA57.td index ade03f23f8c7..9f566d1c7079 100644 --- a/lib/Target/AArch64/AArch64SchedA57.td +++ b/lib/Target/AArch64/AArch64SchedA57.td @@ -1,9 +1,8 @@ //=- AArch64SchedA57.td - ARM Cortex-A57 Scheduling Defs -----*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,7 +31,7 @@ def CortexA57Model : SchedMachineModel { let LoopMicroOpBufferSize = 16; let CompleteModel = 1; - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; } //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64SchedA57WriteRes.td b/lib/Target/AArch64/AArch64SchedA57WriteRes.td index 55005e1d9ed1..987ed3c4ebfb 100644 --- a/lib/Target/AArch64/AArch64SchedA57WriteRes.td +++ b/lib/Target/AArch64/AArch64SchedA57WriteRes.td @@ -1,9 +1,8 @@ //=- AArch64SchedA57WriteRes.td - ARM Cortex-A57 Write Res ---*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64SchedCyclone.td b/lib/Target/AArch64/AArch64SchedCyclone.td index 7a474ba8ef9b..798ecb7508c0 100644 --- a/lib/Target/AArch64/AArch64SchedCyclone.td +++ b/lib/Target/AArch64/AArch64SchedCyclone.td @@ -1,9 +1,8 @@ //=- AArch64SchedCyclone.td - Cyclone Scheduling Definitions -*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,7 +18,7 @@ def CycloneModel : SchedMachineModel { let MispredictPenalty = 16; // 14-19 cycles are typical. let CompleteModel = 1; - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; } //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64SchedExynosM1.td b/lib/Target/AArch64/AArch64SchedExynosM1.td index f757d53b6c1c..f1e76e2c20d3 100644 --- a/lib/Target/AArch64/AArch64SchedExynosM1.td +++ b/lib/Target/AArch64/AArch64SchedExynosM1.td @@ -1,9 +1,8 @@ //=- AArch64SchedExynosM1.td - Samsung Exynos M1 Sched Defs --*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,7 @@ def ExynosM1Model : SchedMachineModel { let MispredictPenalty = 14; // Minimum branch misprediction penalty. let CompleteModel = 1; // Use the default model otherwise. - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; } //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64SchedExynosM3.td b/lib/Target/AArch64/AArch64SchedExynosM3.td index 15935088a17e..c9d29d75d9db 100644 --- a/lib/Target/AArch64/AArch64SchedExynosM3.td +++ b/lib/Target/AArch64/AArch64SchedExynosM3.td @@ -1,9 +1,8 @@ //=- AArch64SchedExynosM3.td - Samsung Exynos M3 Sched Defs --*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,7 @@ def ExynosM3Model : SchedMachineModel { let MispredictPenalty = 16; // Minimum branch misprediction penalty. let CompleteModel = 1; // Use the default model otherwise. - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; } //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64SchedExynosM4.td b/lib/Target/AArch64/AArch64SchedExynosM4.td index 4d892465b3f2..c8bf05f16131 100644 --- a/lib/Target/AArch64/AArch64SchedExynosM4.td +++ b/lib/Target/AArch64/AArch64SchedExynosM4.td @@ -1,9 +1,8 @@ //=- AArch64SchedExynosM4.td - Samsung Exynos M4 Sched Defs --*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,7 @@ def ExynosM4Model : SchedMachineModel { let MispredictPenalty = 16; // Minimum branch misprediction penalty. let CompleteModel = 1; // Use the default model otherwise. - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; } //===----------------------------------------------------------------------===// @@ -239,7 +238,6 @@ def M4WriteNEONK : SchedWriteRes<[M4UnitNSHF, M4UnitS0]> { let Latency = 5; let NumMicroOps = 2; } def M4WriteNEONL : SchedWriteRes<[M4UnitNMUL]> { let Latency = 3; } -def M4WriteNEONM : SchedWriteRes<[M4UnitNMUL]> { let Latency = 3; } def M4WriteNEONN : SchedWriteRes<[M4UnitNMSC, M4UnitNMSC]> { let Latency = 5; let NumMicroOps = 2; } @@ -480,8 +478,6 @@ def M4WriteCOPY : SchedWriteVariant<[SchedVar<ExynosFPPred, [M4WriteNALU1]>, SchedVar<NoSchedPred, [M4WriteZ0]>]>; def M4WriteMOVI : SchedWriteVariant<[SchedVar<IsZeroFPIdiomPred, [M4WriteZ0]>, SchedVar<NoSchedPred, [M4WriteNALU1]>]>; -def M4WriteMULL : SchedWriteVariant<[SchedVar<ExynosLongVectorUpperPred, [M4WriteNEONM]>, - SchedVar<NoSchedPred, [M4WriteNMUL3]>]>; // Fast forwarding. def M4ReadAESM1 : SchedReadAdvance<+1, [M4WriteNCRY1]>; @@ -489,7 +485,8 @@ def M4ReadFMACM1 : SchedReadAdvance<+1, [M4WriteFMAC4, M4WriteFMAC4H, M4WriteFMAC5]>; def M4ReadNMULM1 : SchedReadAdvance<+1, [M4WriteNMUL3]>; -def M4ReadMULLP2 : SchedReadAdvance<-2, [M4WriteNEONM]>; +def M4ReadNMULP2 : SchedReadAdvance<-2, [M4WriteNMUL3]>; + //===----------------------------------------------------------------------===// // Coarse scheduling model. @@ -662,10 +659,8 @@ def : InstRW<[M4WriteNEONK], (instregex "^FMOVDXHighr")>; def : InstRW<[M4WriteFCVT3H], (instregex "^F(RECP|RSQRT)Ev1f16")>; def : InstRW<[M4WriteFCVT3], (instregex "^F(RECP|RSQRT)Ev1i(32|64)")>; def : InstRW<[M4WriteNMSC1], (instregex "^FRECPXv1")>; -def : InstRW<[M4WriteFMAC4H, - M4ReadFMACM1], (instregex "^F(RECP|RSQRT)S16")>; -def : InstRW<[M4WriteFMAC4, - M4ReadFMACM1], (instregex "^F(RECP|RSQRT)S(32|64)")>; +def : InstRW<[M4WriteFMAC4H], (instregex "^F(RECP|RSQRT)S16")>; +def : InstRW<[M4WriteFMAC4], (instregex "^F(RECP|RSQRT)S(32|64)")>; // FP load instructions. def : InstRW<[WriteVLD], (instregex "^LDR[SDQ]l")>; @@ -736,14 +731,20 @@ def : InstRW<[M4WriteNALU1], (instregex "^(AND|BIC|EOR|NOT|ORN|ORR)v")>; def : InstRW<[M4WriteNMSC1], (instregex "^[SU](MIN|MAX)v")>; def : InstRW<[M4WriteNMSC2], (instregex "^[SU](MIN|MAX)Pv")>; def : InstRW<[M4WriteNHAD3], (instregex "^[SU](MIN|MAX)Vv")>; -def : InstRW<[M4WriteNMUL3], (instregex "^(SQR?D)?MULH?v")>; def : InstRW<[M4WriteNMUL3, M4ReadNMULM1], (instregex "^ML[AS]v")>; -def : InstRW<[M4WriteNMUL3], (instregex "^SQRDML[AS]H")>; -def : InstRW<[M4WriteMULL, - M4ReadMULLP2], (instregex "^(S|U|SQD)ML[AS]Lv")>; -def : InstRW<[M4WriteMULL, - M4ReadMULLP2], (instregex "^(S|U|SQD)MULLv")>; +def : InstRW<[M4WriteNMUL3, + M4ReadNMULM1], (instregex "^(SQR?D)?MULH?v")>; +def : InstRW<[M4WriteNMUL3, + M4ReadNMULM1], (instregex "^SQRDML[AS]H")>; +def : InstRW<[M4WriteNMUL3, + M4ReadNMULM1], (instregex "^(S|U|SQD)ML[AS]L(v1(i32|i64)|v2i32|v4i16|v8i8)")>; +def : InstRW<[M4WriteNMUL3, + M4ReadNMULP2], (instregex "^(S|U|SQD)ML[AS]L(v4i32|v8i16|v16i8)")>; +def : InstRW<[M4WriteNMUL3, + M4ReadNMULM1], (instregex "^(S|U|SQD)MULL(v1(i32|i64)|v2i32|v4i16|v8i8)")>; +def : InstRW<[M4WriteNMUL3, + M4ReadNMULP2], (instregex "^(S|U|SQD)MULL(v4i32|v8i16|v16i8)")>; def : InstRW<[M4WriteNMUL3], (instregex "^[SU]DOT(lane)?v")>; def : InstRW<[M4WriteNHAD3], (instregex "^[SU]ADALPv")>; def : InstRW<[M4WriteNSHT4A], (instregex "^[SU]R?SRA[dv]")>; @@ -808,10 +809,8 @@ def : InstRW<[M4WriteNALU1], (instregex "^FMOVv.f(32|64)")>; def : InstRW<[M4WriteFCVT3H], (instregex "^F(RECP|RSQRT)Ev[248]f16")>; def : InstRW<[M4WriteFCVT3], (instregex "^F(RECP|RSQRT)Ev[248]f(32|64)")>; def : InstRW<[M4WriteFCVT3], (instregex "^U(RECP|RSQRT)Ev[24]i32")>; -def : InstRW<[M4WriteFMAC4H, - M4ReadFMACM1], (instregex "^F(RECP|RSQRT)Sv.f16")>; -def : InstRW<[M4WriteFMAC4, - M4ReadFMACM1], (instregex "^F(RECP|RSQRT)Sv.f(32|64)")>; +def : InstRW<[M4WriteFMAC4H], (instregex "^F(RECP|RSQRT)Sv.f16")>; +def : InstRW<[M4WriteFMAC4], (instregex "^F(RECP|RSQRT)Sv.f(32|64)")>; def : InstRW<[M4WriteNSHF1], (instregex "^REV(16|32|64)v")>; def : InstRW<[M4WriteNSHFA], (instregex "^TB[LX]v(8|16)i8One")>; def : InstRW<[M4WriteNSHFB], (instregex "^TB[LX]v(8|16)i8Two")>; diff --git a/lib/Target/AArch64/AArch64SchedFalkor.td b/lib/Target/AArch64/AArch64SchedFalkor.td index 84825458e47c..92d03963de57 100644 --- a/lib/Target/AArch64/AArch64SchedFalkor.td +++ b/lib/Target/AArch64/AArch64SchedFalkor.td @@ -1,9 +1,8 @@ //==- AArch64SchedFalkor.td - Falkor Scheduling Definitions -*- tablegen -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -24,7 +23,7 @@ def FalkorModel : SchedMachineModel { let MispredictPenalty = 11; // Minimum branch misprediction penalty. let CompleteModel = 1; - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; // FIXME: Remove when all errors have been fixed. let FullInstRWOverlapCheck = 0; diff --git a/lib/Target/AArch64/AArch64SchedFalkorDetails.td b/lib/Target/AArch64/AArch64SchedFalkorDetails.td index ff14e639d1a5..697a0f69c58c 100644 --- a/lib/Target/AArch64/AArch64SchedFalkorDetails.td +++ b/lib/Target/AArch64/AArch64SchedFalkorDetails.td @@ -1,9 +1,8 @@ //==- AArch64SchedFalkorDetails.td - Falkor Scheduling Defs -*- tablegen -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64SchedKryo.td b/lib/Target/AArch64/AArch64SchedKryo.td index 68de3e077c96..0e1a24103121 100644 --- a/lib/Target/AArch64/AArch64SchedKryo.td +++ b/lib/Target/AArch64/AArch64SchedKryo.td @@ -1,9 +1,8 @@ //==- AArch64SchedKryo.td - Qualcomm Kryo Scheduling Defs ---*- tablegen -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,7 @@ def KryoModel : SchedMachineModel { let LoopMicroOpBufferSize = 16; let CompleteModel = 1; - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; // FIXME: Remove when all errors have been fixed. let FullInstRWOverlapCheck = 0; diff --git a/lib/Target/AArch64/AArch64SchedKryoDetails.td b/lib/Target/AArch64/AArch64SchedKryoDetails.td index cf4cdabb8cbf..4c60992e6351 100644 --- a/lib/Target/AArch64/AArch64SchedKryoDetails.td +++ b/lib/Target/AArch64/AArch64SchedKryoDetails.td @@ -1,9 +1,8 @@ //=- AArch64SchedKryoDetails.td - QC Kryo Scheduling Defs ----*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64SchedPredExynos.td b/lib/Target/AArch64/AArch64SchedPredExynos.td index 48c54230e9d8..0c1d82d354c0 100644 --- a/lib/Target/AArch64/AArch64SchedPredExynos.td +++ b/lib/Target/AArch64/AArch64SchedPredExynos.td @@ -1,9 +1,8 @@ //===- AArch64SchedPredExynos.td - AArch64 Sched Preds -----*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -103,17 +102,6 @@ def ExynosScaledIdxPred : MCSchedPredicate<ExynosScaledIdxFn>; // Identify FP instructions. def ExynosFPPred : MCSchedPredicate<CheckAny<[CheckDForm, CheckQForm]>>; -// Identify whether an instruction whose result is a long vector -// operates on the upper half of the input registers. -def ExynosLongVectorUpperFn : TIIPredicate< - "isExynosLongVectorUpper", - MCOpcodeSwitchStatement< - [MCOpcodeSwitchCase< - IsLongVectorUpperOp.ValidOpcodes, - MCReturnStatement<TruePred>>], - MCReturnStatement<FalsePred>>>; -def ExynosLongVectorUpperPred : MCSchedPredicate<ExynosLongVectorUpperFn>; - // Identify 128-bit NEON instructions. def ExynosQFormPred : MCSchedPredicate<CheckQForm>; diff --git a/lib/Target/AArch64/AArch64SchedPredicates.td b/lib/Target/AArch64/AArch64SchedPredicates.td index dbaf11fc95dd..0ef0f3f8675a 100644 --- a/lib/Target/AArch64/AArch64SchedPredicates.td +++ b/lib/Target/AArch64/AArch64SchedPredicates.td @@ -1,9 +1,8 @@ //===- AArch64SchedPredicates.td - AArch64 Sched Preds -----*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -268,59 +267,6 @@ def IsStoreRegOffsetOp : CheckOpcode<[STRBBroW, STRBBroX, def IsLoadStoreRegOffsetOp : CheckOpcode<!listconcat(IsLoadRegOffsetOp.ValidOpcodes, IsStoreRegOffsetOp.ValidOpcodes)>; -// Identify whether an instruction whose result is a long vector -// operates on the upper half of the input registers. -def IsLongVectorUpperOp : CheckOpcode<[FCVTLv8i16, FCVTLv4i32, - FCVTNv8i16, FCVTNv4i32, - FCVTXNv4f32, - PMULLv16i8, PMULLv2i64, - RADDHNv8i16_v16i8, RADDHNv4i32_v8i16, RADDHNv2i64_v4i32, - RSHRNv16i8_shift, RSHRNv8i16_shift, RSHRNv4i32_shift, - RSUBHNv8i16_v16i8, RSUBHNv4i32_v8i16, RSUBHNv2i64_v4i32, - SABALv16i8_v8i16, SABALv8i16_v4i32, SABALv4i32_v2i64, - SABDLv16i8_v8i16, SABDLv8i16_v4i32, SABDLv4i32_v2i64, - SADDLv16i8_v8i16, SADDLv8i16_v4i32, SADDLv4i32_v2i64, - SADDWv16i8_v8i16, SADDWv8i16_v4i32, SADDWv4i32_v2i64, - SHLLv16i8, SHLLv8i16, SHLLv4i32, - SHRNv16i8_shift, SHRNv8i16_shift, SHRNv4i32_shift, - SMLALv16i8_v8i16, SMLALv8i16_v4i32, SMLALv4i32_v2i64, - SMLALv8i16_indexed, SMLALv4i32_indexed, - SMLSLv16i8_v8i16, SMLSLv8i16_v4i32, SMLSLv4i32_v2i64, - SMLSLv8i16_indexed, SMLSLv4i32_indexed, - SMULLv16i8_v8i16, SMULLv8i16_v4i32, SMULLv4i32_v2i64, - SMULLv8i16_indexed, SMULLv4i32_indexed, - SQDMLALv8i16_v4i32, SQDMLALv4i32_v2i64, - SQDMLALv8i16_indexed, SQDMLALv4i32_indexed, - SQDMLSLv8i16_v4i32, SQDMLSLv4i32_v2i64, - SQDMLSLv8i16_indexed, SQDMLSLv4i32_indexed, - SQDMULLv8i16_v4i32, SQDMULLv4i32_v2i64, - SQDMULLv8i16_indexed, SQDMULLv4i32_indexed, - SQRSHRNv16i8_shift, SQRSHRNv8i16_shift, SQRSHRNv4i32_shift, - SQRSHRUNv16i8_shift, SQRSHRUNv8i16_shift, SQRSHRUNv4i32_shift, - SQSHRNv16i8_shift, SQSHRNv8i16_shift, SQSHRNv4i32_shift, - SQSHRUNv16i8_shift, SQSHRUNv8i16_shift, SQSHRUNv4i32_shift, - SQXTNv16i8, SQXTNv8i16, SQXTNv4i32, - SQXTUNv16i8, SQXTUNv8i16, SQXTUNv4i32, - SSHLLv16i8_shift, SSHLLv8i16_shift, SSHLLv4i32_shift, - SSUBLv16i8_v8i16, SSUBLv8i16_v4i32, SSUBLv4i32_v2i64, - SSUBWv16i8_v8i16, SSUBWv8i16_v4i32, SSUBWv4i32_v2i64, - UABALv16i8_v8i16, UABALv8i16_v4i32, UABALv4i32_v2i64, - UABDLv16i8_v8i16, UABDLv8i16_v4i32, UABDLv4i32_v2i64, - UADDLv16i8_v8i16, UADDLv8i16_v4i32, UADDLv4i32_v2i64, - UADDWv16i8_v8i16, UADDWv8i16_v4i32, UADDWv4i32_v2i64, - UMLALv16i8_v8i16, UMLALv8i16_v4i32, UMLALv4i32_v2i64, - UMLALv8i16_indexed, UMLALv4i32_indexed, - UMLSLv16i8_v8i16, UMLSLv8i16_v4i32, UMLSLv4i32_v2i64, - UMLSLv8i16_indexed, UMLSLv4i32_indexed, - UMULLv16i8_v8i16, UMULLv8i16_v4i32, UMULLv4i32_v2i64, - UMULLv8i16_indexed, UMULLv4i32_indexed, - UQSHRNv16i8_shift, UQSHRNv8i16_shift, UQSHRNv4i32_shift, - UQXTNv16i8, UQXTNv8i16, UQXTNv4i32, - USHLLv16i8_shift, USHLLv8i16_shift, USHLLv4i32_shift, - USUBLv16i8_v8i16, USUBLv8i16_v4i32, USUBLv4i32_v2i64, - USUBWv16i8_v8i16, USUBWv8i16_v4i32, USUBWv4i32_v2i64, - XTNv16i8, XTNv8i16, XTNv4i32]>; - // Target predicates. // Identify an instruction that effectively transfers a register to another. diff --git a/lib/Target/AArch64/AArch64SchedThunderX.td b/lib/Target/AArch64/AArch64SchedThunderX.td index fbbd3850d0fd..3b6aecf5c035 100644 --- a/lib/Target/AArch64/AArch64SchedThunderX.td +++ b/lib/Target/AArch64/AArch64SchedThunderX.td @@ -1,9 +1,8 @@ //==- AArch64SchedThunderX.td - Cavium ThunderX T8X Scheduling Definitions -*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,7 +25,7 @@ def ThunderXT8XModel : SchedMachineModel { let PostRAScheduler = 1; // Use PostRA scheduler. let CompleteModel = 1; - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; // FIXME: Remove when all errors have been fixed. let FullInstRWOverlapCheck = 0; diff --git a/lib/Target/AArch64/AArch64SchedThunderX2T99.td b/lib/Target/AArch64/AArch64SchedThunderX2T99.td index bee3392b6d3b..674ea19b082f 100644 --- a/lib/Target/AArch64/AArch64SchedThunderX2T99.td +++ b/lib/Target/AArch64/AArch64SchedThunderX2T99.td @@ -1,9 +1,8 @@ //=- AArch64SchedThunderX2T99.td - Cavium ThunderX T99 ---*- tablegen -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,7 +25,7 @@ def ThunderX2T99Model : SchedMachineModel { let PostRAScheduler = 1; // Using PostRA sched. let CompleteModel = 1; - list<Predicate> UnsupportedFeatures = [HasSVE]; + list<Predicate> UnsupportedFeatures = SVEUnsupported.F; // FIXME: Remove when all errors have been fixed. let FullInstRWOverlapCheck = 0; diff --git a/lib/Target/AArch64/AArch64Schedule.td b/lib/Target/AArch64/AArch64Schedule.td index f55ba4d42fce..49c0c1782236 100644 --- a/lib/Target/AArch64/AArch64Schedule.td +++ b/lib/Target/AArch64/AArch64Schedule.td @@ -1,9 +1,8 @@ //==-- AArch64Schedule.td - AArch64 Scheduling Definitions -*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp index a719d47618e5..60dbace03ca6 100644 --- a/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp +++ b/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp @@ -1,9 +1,8 @@ //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -57,3 +56,91 @@ bool AArch64SelectionDAGInfo::generateFMAsInMachineCombiner( CodeGenOpt::Level OptLevel) const { return OptLevel >= CodeGenOpt::Aggressive; } + +static const int kSetTagLoopThreshold = 176; + +static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, + SDValue Chain, SDValue Ptr, uint64_t ObjSize, + const MachineMemOperand *BaseMemOperand, + bool ZeroData) { + MachineFunction &MF = DAG.getMachineFunction(); + unsigned ObjSizeScaled = ObjSize / 16; + + SDValue TagSrc = Ptr; + if (Ptr.getOpcode() == ISD::FrameIndex) { + int FI = cast<FrameIndexSDNode>(Ptr)->getIndex(); + Ptr = DAG.getTargetFrameIndex(FI, MVT::i64); + // A frame index operand may end up as [SP + offset] => it is fine to use SP + // register as the tag source. + TagSrc = DAG.getRegister(AArch64::SP, MVT::i64); + } + + const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG; + const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G; + + SmallVector<SDValue, 8> OutChains; + unsigned OffsetScaled = 0; + while (OffsetScaled < ObjSizeScaled) { + if (ObjSizeScaled - OffsetScaled >= 2) { + SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl); + SDValue St = DAG.getMemIntrinsicNode( + OpCode2, dl, DAG.getVTList(MVT::Other), + {Chain, TagSrc, AddrNode}, + MVT::v4i64, + MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2)); + OffsetScaled += 2; + OutChains.push_back(St); + continue; + } + + if (ObjSizeScaled - OffsetScaled > 0) { + SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl); + SDValue St = DAG.getMemIntrinsicNode( + OpCode1, dl, DAG.getVTList(MVT::Other), + {Chain, TagSrc, AddrNode}, + MVT::v2i64, + MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16)); + OffsetScaled += 1; + OutChains.push_back(St); + } + } + + SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); + return Res; +} + +SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag( + SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, + SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const { + uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue(); + assert(ObjSize % 16 == 0); + + MachineFunction &MF = DAG.getMachineFunction(); + MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand( + DstPtrInfo, MachineMemOperand::MOStore, ObjSize, 16); + + bool UseSetTagRangeLoop = + kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold; + if (!UseSetTagRangeLoop) + return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand, + ZeroData); + + if (ObjSize % 32 != 0) { + SDNode *St1 = DAG.getMachineNode( + ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex, dl, + {MVT::i64, MVT::Other}, + {Addr, Addr, DAG.getTargetConstant(1, dl, MVT::i64), Chain}); + DAG.setNodeMemRefs(cast<MachineSDNode>(St1), {BaseMemOperand}); + ObjSize -= 16; + Addr = SDValue(St1, 0); + Chain = SDValue(St1, 1); + } + + const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other}; + SDValue Ops[] = {DAG.getConstant(ObjSize, dl, MVT::i64), Addr, Chain}; + SDNode *St = DAG.getMachineNode( + ZeroData ? AArch64::STZGloop : AArch64::STGloop, dl, ResTys, Ops); + + DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand}); + return SDValue(St, 2); +} diff --git a/lib/Target/AArch64/AArch64SelectionDAGInfo.h b/lib/Target/AArch64/AArch64SelectionDAGInfo.h index 7e4f11091226..d0967fb973cc 100644 --- a/lib/Target/AArch64/AArch64SelectionDAGInfo.h +++ b/lib/Target/AArch64/AArch64SelectionDAGInfo.h @@ -1,9 +1,8 @@ //===-- AArch64SelectionDAGInfo.h - AArch64 SelectionDAG Info ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -24,6 +23,10 @@ public: SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVolatile, MachinePointerInfo DstPtrInfo) const override; + SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, + SDValue Chain, SDValue Op1, SDValue Op2, + MachinePointerInfo DstPtrInfo, + bool ZeroData) const override; bool generateFMAsInMachineCombiner(CodeGenOpt::Level OptLevel) const override; }; } diff --git a/lib/Target/AArch64/AArch64SpeculationHardening.cpp b/lib/Target/AArch64/AArch64SpeculationHardening.cpp index e9699b0367d3..3087e6ce441d 100644 --- a/lib/Target/AArch64/AArch64SpeculationHardening.cpp +++ b/lib/Target/AArch64/AArch64SpeculationHardening.cpp @@ -1,9 +1,8 @@ //===- AArch64SpeculationHardening.cpp - Harden Against Missspeculation --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -103,6 +102,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/IR/DebugLoc.h" #include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" @@ -146,25 +146,31 @@ private: BitVector RegsAlreadyMasked; bool functionUsesHardeningRegister(MachineFunction &MF) const; - bool instrumentControlFlow(MachineBasicBlock &MBB); + bool instrumentControlFlow(MachineBasicBlock &MBB, + bool &UsesFullSpeculationBarrier); bool endsWithCondControlFlow(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, AArch64CC::CondCode &CondCode) const; void insertTrackingCode(MachineBasicBlock &SplitEdgeBB, AArch64CC::CondCode &CondCode, DebugLoc DL) const; - void insertSPToRegTaintPropagation(MachineBasicBlock *MBB, + void insertSPToRegTaintPropagation(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; - void insertRegToSPTaintPropagation(MachineBasicBlock *MBB, + void insertRegToSPTaintPropagation(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned TmpReg) const; + void insertFullSpeculationBarrier(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + DebugLoc DL) const; bool slhLoads(MachineBasicBlock &MBB); bool makeGPRSpeculationSafe(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineInstr &MI, unsigned Reg); - bool lowerSpeculationSafeValuePseudos(MachineBasicBlock &MBB); + bool lowerSpeculationSafeValuePseudos(MachineBasicBlock &MBB, + bool UsesFullSpeculationBarrier); bool expandSpeculationSafeValue(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI); + MachineBasicBlock::iterator MBBI, + bool UsesFullSpeculationBarrier); bool insertCSDB(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL); }; @@ -207,15 +213,19 @@ bool AArch64SpeculationHardening::endsWithCondControlFlow( return true; } +void AArch64SpeculationHardening::insertFullSpeculationBarrier( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + DebugLoc DL) const { + // A full control flow speculation barrier consists of (DSB SYS + ISB) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::DSB)).addImm(0xf); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ISB)).addImm(0xf); +} + void AArch64SpeculationHardening::insertTrackingCode( MachineBasicBlock &SplitEdgeBB, AArch64CC::CondCode &CondCode, DebugLoc DL) const { if (UseControlFlowSpeculationBarrier) { - // insert full control flow speculation barrier (DSB SYS + ISB) - BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::ISB)) - .addImm(0xf); - BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::DSB)) - .addImm(0xf); + insertFullSpeculationBarrier(SplitEdgeBB, SplitEdgeBB.begin(), DL); } else { BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::CSELXr)) .addDef(MisspeculatingTaintReg) @@ -227,7 +237,7 @@ void AArch64SpeculationHardening::insertTrackingCode( } bool AArch64SpeculationHardening::instrumentControlFlow( - MachineBasicBlock &MBB) { + MachineBasicBlock &MBB, bool &UsesFullSpeculationBarrier) { LLVM_DEBUG(dbgs() << "Instrument control flow tracking on MBB: " << MBB); bool Modified = false; @@ -263,55 +273,105 @@ bool AArch64SpeculationHardening::instrumentControlFlow( } // Perform correct code generation around function calls and before returns. - { - SmallVector<MachineInstr *, 4> ReturnInstructions; - SmallVector<MachineInstr *, 4> CallInstructions; + // The below variables record the return/terminator instructions and the call + // instructions respectively; including which register is available as a + // temporary register just before the recorded instructions. + SmallVector<std::pair<MachineInstr *, unsigned>, 4> ReturnInstructions; + SmallVector<std::pair<MachineInstr *, unsigned>, 4> CallInstructions; + // if a temporary register is not available for at least one of the + // instructions for which we need to transfer taint to the stack pointer, we + // need to insert a full speculation barrier. + // TmpRegisterNotAvailableEverywhere tracks that condition. + bool TmpRegisterNotAvailableEverywhere = false; - for (MachineInstr &MI : MBB) { - if (MI.isReturn()) - ReturnInstructions.push_back(&MI); - else if (MI.isCall()) - CallInstructions.push_back(&MI); - } + RegScavenger RS; + RS.enterBasicBlock(MBB); - Modified |= - (ReturnInstructions.size() > 0) || (CallInstructions.size() > 0); + for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); I++) { + MachineInstr &MI = *I; + if (!MI.isReturn() && !MI.isCall()) + continue; + + // The RegScavenger represents registers available *after* the MI + // instruction pointed to by RS.getCurrentPosition(). + // We need to have a register that is available *before* the MI is executed. + if (I != MBB.begin()) + RS.forward(std::prev(I)); + // FIXME: The below just finds *a* unused register. Maybe code could be + // optimized more if this looks for the register that isn't used for the + // longest time around this place, to enable more scheduling freedom. Not + // sure if that would actually result in a big performance difference + // though. Maybe RegisterScavenger::findSurvivorBackwards has some logic + // already to do this - but it's unclear if that could easily be used here. + unsigned TmpReg = RS.FindUnusedReg(&AArch64::GPR64commonRegClass); + LLVM_DEBUG(dbgs() << "RS finds " + << ((TmpReg == 0) ? "no register " : "register "); + if (TmpReg != 0) dbgs() << printReg(TmpReg, TRI) << " "; + dbgs() << "to be available at MI " << MI); + if (TmpReg == 0) + TmpRegisterNotAvailableEverywhere = true; + if (MI.isReturn()) + ReturnInstructions.push_back({&MI, TmpReg}); + else if (MI.isCall()) + CallInstructions.push_back({&MI, TmpReg}); + } - for (MachineInstr *Return : ReturnInstructions) - insertRegToSPTaintPropagation(Return->getParent(), Return, AArch64::X17); - for (MachineInstr *Call : CallInstructions) { + if (TmpRegisterNotAvailableEverywhere) { + // When a temporary register is not available everywhere in this basic + // basic block where a propagate-taint-to-sp operation is needed, just + // emit a full speculation barrier at the start of this basic block, which + // renders the taint/speculation tracking in this basic block unnecessary. + insertFullSpeculationBarrier(MBB, MBB.begin(), + (MBB.begin())->getDebugLoc()); + UsesFullSpeculationBarrier = true; + Modified = true; + } else { + for (auto MI_Reg : ReturnInstructions) { + assert(MI_Reg.second != 0); + LLVM_DEBUG( + dbgs() + << " About to insert Reg to SP taint propagation with temp register " + << printReg(MI_Reg.second, TRI) + << " on instruction: " << *MI_Reg.first); + insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second); + Modified = true; + } + + for (auto MI_Reg : CallInstructions) { + assert(MI_Reg.second != 0); + LLVM_DEBUG(dbgs() << " About to insert Reg to SP and back taint " + "propagation with temp register " + << printReg(MI_Reg.second, TRI) + << " around instruction: " << *MI_Reg.first); // Just after the call: - MachineBasicBlock::iterator i = Call; - i++; - insertSPToRegTaintPropagation(Call->getParent(), i); + insertSPToRegTaintPropagation( + MBB, std::next((MachineBasicBlock::iterator)MI_Reg.first)); // Just before the call: - insertRegToSPTaintPropagation(Call->getParent(), Call, AArch64::X17); + insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second); + Modified = true; } } - return Modified; } void AArch64SpeculationHardening::insertSPToRegTaintPropagation( - MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) const { + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { // If full control flow speculation barriers are used, emit a control flow // barrier to block potential miss-speculation in flight coming in to this // function. if (UseControlFlowSpeculationBarrier) { - // insert full control flow speculation barrier (DSB SYS + ISB) - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::DSB)).addImm(0xf); - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ISB)).addImm(0xf); + insertFullSpeculationBarrier(MBB, MBBI, DebugLoc()); return; } // CMP SP, #0 === SUBS xzr, SP, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::SUBSXri)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::SUBSXri)) .addDef(AArch64::XZR) .addUse(AArch64::SP) .addImm(0) .addImm(0); // no shift // CSETM x16, NE === CSINV x16, xzr, xzr, EQ - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::CSINVXr)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::CSINVXr)) .addDef(MisspeculatingTaintReg) .addUse(AArch64::XZR) .addUse(AArch64::XZR) @@ -319,7 +379,7 @@ void AArch64SpeculationHardening::insertSPToRegTaintPropagation( } void AArch64SpeculationHardening::insertRegToSPTaintPropagation( - MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned TmpReg) const { // If full control flow speculation barriers are used, there will not be // miss-speculation when returning from this function, and therefore, also @@ -328,19 +388,19 @@ void AArch64SpeculationHardening::insertRegToSPTaintPropagation( return; // mov Xtmp, SP === ADD Xtmp, SP, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) .addDef(TmpReg) .addUse(AArch64::SP) .addImm(0) .addImm(0); // no shift // and Xtmp, Xtmp, TaintReg === AND Xtmp, Xtmp, TaintReg, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ANDXrs)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ANDXrs)) .addDef(TmpReg, RegState::Renamable) .addUse(TmpReg, RegState::Kill | RegState::Renamable) .addUse(MisspeculatingTaintReg, RegState::Kill) .addImm(0); // mov SP, Xtmp === ADD SP, Xtmp, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) .addDef(AArch64::SP) .addUse(TmpReg, RegState::Kill) .addImm(0) @@ -484,7 +544,8 @@ bool AArch64SpeculationHardening::slhLoads(MachineBasicBlock &MBB) { /// \brief If MBBI references a pseudo instruction that should be expanded /// here, do the expansion and return true. Otherwise return false. bool AArch64SpeculationHardening::expandSpeculationSafeValue( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + bool UsesFullSpeculationBarrier) { MachineInstr &MI = *MBBI; unsigned Opcode = MI.getOpcode(); bool Is64Bit = true; @@ -499,7 +560,7 @@ bool AArch64SpeculationHardening::expandSpeculationSafeValue( // Just remove the SpeculationSafe pseudo's if control flow // miss-speculation isn't happening because we're already inserting barriers // to guarantee that. - if (!UseControlFlowSpeculationBarrier) { + if (!UseControlFlowSpeculationBarrier && !UsesFullSpeculationBarrier) { unsigned DstReg = MI.getOperand(0).getReg(); unsigned SrcReg = MI.getOperand(1).getReg(); // Mark this register and all its aliasing registers as needing to be @@ -537,7 +598,7 @@ bool AArch64SpeculationHardening::insertCSDB(MachineBasicBlock &MBB, } bool AArch64SpeculationHardening::lowerSpeculationSafeValuePseudos( - MachineBasicBlock &MBB) { + MachineBasicBlock &MBB, bool UsesFullSpeculationBarrier) { bool Modified = false; RegsNeedingCSDBBeforeUse.reset(); @@ -572,15 +633,16 @@ bool AArch64SpeculationHardening::lowerSpeculationSafeValuePseudos( break; } - if (NeedToEmitBarrier) + if (NeedToEmitBarrier && !UsesFullSpeculationBarrier) Modified |= insertCSDB(MBB, MBBI, DL); - Modified |= expandSpeculationSafeValue(MBB, MBBI); + Modified |= + expandSpeculationSafeValue(MBB, MBBI, UsesFullSpeculationBarrier); MBBI = NMBBI; } - if (RegsNeedingCSDBBeforeUse.any()) + if (RegsNeedingCSDBBeforeUse.any() && !UsesFullSpeculationBarrier) Modified |= insertCSDB(MBB, MBBI, DL); return Modified; @@ -609,7 +671,7 @@ bool AArch64SpeculationHardening::runOnMachineFunction(MachineFunction &MF) { Modified |= slhLoads(MBB); } - // 2.a Add instrumentation code to function entry and exits. + // 2. Add instrumentation code to function entry and exits. LLVM_DEBUG( dbgs() << "***** AArch64SpeculationHardening - track control flow *****\n"); @@ -620,17 +682,15 @@ bool AArch64SpeculationHardening::runOnMachineFunction(MachineFunction &MF) { EntryBlocks.push_back(LPI.LandingPadBlock); for (auto Entry : EntryBlocks) insertSPToRegTaintPropagation( - Entry, Entry->SkipPHIsLabelsAndDebug(Entry->begin())); - - // 2.b Add instrumentation code to every basic block. - for (auto &MBB : MF) - Modified |= instrumentControlFlow(MBB); + *Entry, Entry->SkipPHIsLabelsAndDebug(Entry->begin())); - LLVM_DEBUG(dbgs() << "***** AArch64SpeculationHardening - Lowering " - "SpeculationSafeValue Pseudos *****\n"); - // Step 3: Lower SpeculationSafeValue pseudo instructions. - for (auto &MBB : MF) - Modified |= lowerSpeculationSafeValuePseudos(MBB); + // 3. Add instrumentation code to every basic block. + for (auto &MBB : MF) { + bool UsesFullSpeculationBarrier = false; + Modified |= instrumentControlFlow(MBB, UsesFullSpeculationBarrier); + Modified |= + lowerSpeculationSafeValuePseudos(MBB, UsesFullSpeculationBarrier); + } return Modified; } diff --git a/lib/Target/AArch64/AArch64StackTagging.cpp b/lib/Target/AArch64/AArch64StackTagging.cpp new file mode 100644 index 000000000000..6e99c48bf1d7 --- /dev/null +++ b/lib/Target/AArch64/AArch64StackTagging.cpp @@ -0,0 +1,345 @@ +//===- AArch64StackTagging.cpp - Stack tagging in IR --===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// + +#include "AArch64.h" +#include "AArch64InstrInfo.h" +#include "AArch64Subtarget.h" +#include "AArch64TargetMachine.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Metadata.h" +#include "llvm/Pass.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/Local.h" +#include <cassert> +#include <iterator> +#include <utility> + +using namespace llvm; + +#define DEBUG_TYPE "stack-tagging" + +static constexpr unsigned kTagGranuleSize = 16; + +namespace { +class AArch64StackTagging : public FunctionPass { + struct AllocaInfo { + AllocaInst *AI; + SmallVector<IntrinsicInst *, 2> LifetimeStart; + SmallVector<IntrinsicInst *, 2> LifetimeEnd; + SmallVector<DbgVariableIntrinsic *, 2> DbgVariableIntrinsics; + int Tag; // -1 for non-tagged allocations + }; + +public: + static char ID; // Pass ID, replacement for typeid + + AArch64StackTagging() : FunctionPass(ID) { + initializeAArch64StackTaggingPass(*PassRegistry::getPassRegistry()); + } + + bool isInterestingAlloca(const AllocaInst &AI); + void alignAndPadAlloca(AllocaInfo &Info); + + void tagAlloca(AllocaInst *AI, Instruction *InsertBefore, Value *Ptr, + uint64_t Size); + void untagAlloca(AllocaInst *AI, Instruction *InsertBefore, uint64_t Size); + + Instruction * + insertBaseTaggedPointer(const MapVector<AllocaInst *, AllocaInfo> &Allocas, + const DominatorTree *DT); + bool runOnFunction(Function &F) override; + + StringRef getPassName() const override { return "AArch64 Stack Tagging"; } + +private: + Function *F; + Function *SetTagFunc; + const DataLayout *DL; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + } +}; + +} // end anonymous namespace + +char AArch64StackTagging::ID = 0; + +INITIALIZE_PASS_BEGIN(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging", + false, false) +INITIALIZE_PASS_END(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging", + false, false) + +FunctionPass *llvm::createAArch64StackTaggingPass() { + return new AArch64StackTagging(); +} + +bool AArch64StackTagging::isInterestingAlloca(const AllocaInst &AI) { + // FIXME: support dynamic allocas + bool IsInteresting = + AI.getAllocatedType()->isSized() && AI.isStaticAlloca() && + // alloca() may be called with 0 size, ignore it. + AI.getAllocationSizeInBits(*DL).getValue() > 0 && + // inalloca allocas are not treated as static, and we don't want + // dynamic alloca instrumentation for them as well. + !AI.isUsedWithInAlloca() && + // swifterror allocas are register promoted by ISel + !AI.isSwiftError(); + return IsInteresting; +} + +void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore, + Value *Ptr, uint64_t Size) { + IRBuilder<> IRB(InsertBefore); + IRB.CreateCall(SetTagFunc, {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)}); +} + +void AArch64StackTagging::untagAlloca(AllocaInst *AI, Instruction *InsertBefore, + uint64_t Size) { + IRBuilder<> IRB(InsertBefore); + IRB.CreateCall(SetTagFunc, {IRB.CreatePointerCast(AI, IRB.getInt8PtrTy()), + ConstantInt::get(IRB.getInt64Ty(), Size)}); +} + +Instruction *AArch64StackTagging::insertBaseTaggedPointer( + const MapVector<AllocaInst *, AllocaInfo> &Allocas, + const DominatorTree *DT) { + BasicBlock *PrologueBB = nullptr; + // Try sinking IRG as deep as possible to avoid hurting shrink wrap. + for (auto &I : Allocas) { + const AllocaInfo &Info = I.second; + AllocaInst *AI = Info.AI; + if (Info.Tag < 0) + continue; + if (!PrologueBB) { + PrologueBB = AI->getParent(); + continue; + } + PrologueBB = DT->findNearestCommonDominator(PrologueBB, AI->getParent()); + } + assert(PrologueBB); + + IRBuilder<> IRB(&PrologueBB->front()); + Function *IRG_SP = + Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_irg_sp); + Instruction *Base = + IRB.CreateCall(IRG_SP, {Constant::getNullValue(IRB.getInt64Ty())}); + Base->setName("basetag"); + return Base; +} + +void AArch64StackTagging::alignAndPadAlloca(AllocaInfo &Info) { + unsigned NewAlignment = std::max(Info.AI->getAlignment(), kTagGranuleSize); + Info.AI->setAlignment(NewAlignment); + + uint64_t Size = Info.AI->getAllocationSizeInBits(*DL).getValue() / 8; + uint64_t AlignedSize = alignTo(Size, kTagGranuleSize); + if (Size == AlignedSize) + return; + + // Add padding to the alloca. + Type *AllocatedType = + Info.AI->isArrayAllocation() + ? ArrayType::get( + Info.AI->getAllocatedType(), + dyn_cast<ConstantInt>(Info.AI->getArraySize())->getZExtValue()) + : Info.AI->getAllocatedType(); + Type *PaddingType = + ArrayType::get(Type::getInt8Ty(F->getContext()), AlignedSize - Size); + Type *TypeWithPadding = StructType::get(AllocatedType, PaddingType); + auto *NewAI = new AllocaInst( + TypeWithPadding, Info.AI->getType()->getAddressSpace(), nullptr, "", Info.AI); + NewAI->takeName(Info.AI); + NewAI->setAlignment(Info.AI->getAlignment()); + NewAI->setUsedWithInAlloca(Info.AI->isUsedWithInAlloca()); + NewAI->setSwiftError(Info.AI->isSwiftError()); + NewAI->copyMetadata(*Info.AI); + + auto *NewPtr = new BitCastInst(NewAI, Info.AI->getType(), "", Info.AI); + Info.AI->replaceAllUsesWith(NewPtr); + Info.AI->eraseFromParent(); + Info.AI = NewAI; +} + +// FIXME: check for MTE extension +bool AArch64StackTagging::runOnFunction(Function &Fn) { + if (!Fn.hasFnAttribute(Attribute::SanitizeMemTag)) + return false; + + F = &Fn; + DL = &Fn.getParent()->getDataLayout(); + + MapVector<AllocaInst *, AllocaInfo> Allocas; // need stable iteration order + SmallVector<Instruction *, 8> RetVec; + DenseMap<Value *, AllocaInst *> AllocaForValue; + SmallVector<Instruction *, 4> UnrecognizedLifetimes; + + for (auto &BB : *F) { + for (BasicBlock::iterator IT = BB.begin(); IT != BB.end(); ++IT) { + Instruction *I = &*IT; + if (auto *AI = dyn_cast<AllocaInst>(I)) { + Allocas[AI].AI = AI; + continue; + } + + if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(I)) { + if (auto *AI = + dyn_cast_or_null<AllocaInst>(DVI->getVariableLocation())) { + Allocas[AI].DbgVariableIntrinsics.push_back(DVI); + } + continue; + } + + auto *II = dyn_cast<IntrinsicInst>(I); + if (II && (II->getIntrinsicID() == Intrinsic::lifetime_start || + II->getIntrinsicID() == Intrinsic::lifetime_end)) { + AllocaInst *AI = + llvm::findAllocaForValue(II->getArgOperand(1), AllocaForValue); + if (!AI) { + UnrecognizedLifetimes.push_back(I); + continue; + } + if (II->getIntrinsicID() == Intrinsic::lifetime_start) + Allocas[AI].LifetimeStart.push_back(II); + else + Allocas[AI].LifetimeEnd.push_back(II); + } + + if (isa<ReturnInst>(I) || isa<ResumeInst>(I) || isa<CleanupReturnInst>(I)) + RetVec.push_back(I); + } + } + + if (Allocas.empty()) + return false; + + int NextTag = 0; + int NumInterestingAllocas = 0; + for (auto &I : Allocas) { + AllocaInfo &Info = I.second; + assert(Info.AI); + + if (!isInterestingAlloca(*Info.AI)) { + Info.Tag = -1; + continue; + } + + alignAndPadAlloca(Info); + NumInterestingAllocas++; + Info.Tag = NextTag; + NextTag = (NextTag + 1) % 16; + } + + if (NumInterestingAllocas == 0) + return true; + + SetTagFunc = + Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag); + + // Compute DT only if the function has the attribute, there are more than 1 + // interesting allocas, and it is not available for free. + Instruction *Base; + if (NumInterestingAllocas > 1) { + auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>(); + if (DTWP) { + Base = insertBaseTaggedPointer(Allocas, &DTWP->getDomTree()); + } else { + DominatorTree DT(*F); + Base = insertBaseTaggedPointer(Allocas, &DT); + } + } else { + Base = insertBaseTaggedPointer(Allocas, nullptr); + } + + for (auto &I : Allocas) { + const AllocaInfo &Info = I.second; + AllocaInst *AI = Info.AI; + if (Info.Tag < 0) + continue; + + // Replace alloca with tagp(alloca). + IRBuilder<> IRB(Info.AI->getNextNode()); + Function *TagP = Intrinsic::getDeclaration( + F->getParent(), Intrinsic::aarch64_tagp, {Info.AI->getType()}); + Instruction *TagPCall = + IRB.CreateCall(TagP, {Constant::getNullValue(Info.AI->getType()), Base, + ConstantInt::get(IRB.getInt64Ty(), Info.Tag)}); + if (Info.AI->hasName()) + TagPCall->setName(Info.AI->getName() + ".tag"); + Info.AI->replaceAllUsesWith(TagPCall); + TagPCall->setOperand(0, Info.AI); + + if (UnrecognizedLifetimes.empty() && Info.LifetimeStart.size() == 1 && + Info.LifetimeEnd.size() == 1) { + IntrinsicInst *Start = Info.LifetimeStart[0]; + uint64_t Size = + dyn_cast<ConstantInt>(Start->getArgOperand(0))->getZExtValue(); + Size = alignTo(Size, kTagGranuleSize); + tagAlloca(AI, Start->getNextNode(), Start->getArgOperand(1), Size); + untagAlloca(AI, Info.LifetimeEnd[0], Size); + } else { + uint64_t Size = Info.AI->getAllocationSizeInBits(*DL).getValue() / 8; + Value *Ptr = IRB.CreatePointerCast(TagPCall, IRB.getInt8PtrTy()); + tagAlloca(AI, &*IRB.GetInsertPoint(), Ptr, Size); + for (auto &RI : RetVec) { + untagAlloca(AI, RI, Size); + } + // We may have inserted tag/untag outside of any lifetime interval. + // Remove all lifetime intrinsics for this alloca. + for (auto &II : Info.LifetimeStart) + II->eraseFromParent(); + for (auto &II : Info.LifetimeEnd) + II->eraseFromParent(); + } + + // Fixup debug intrinsics to point to the new alloca. + for (auto DVI : Info.DbgVariableIntrinsics) + DVI->setArgOperand( + 0, + MetadataAsValue::get(F->getContext(), LocalAsMetadata::get(Info.AI))); + } + + // If we have instrumented at least one alloca, all unrecognized lifetime + // instrinsics have to go. + for (auto &I : UnrecognizedLifetimes) + I->eraseFromParent(); + + return true; +} diff --git a/lib/Target/AArch64/AArch64StorePairSuppress.cpp b/lib/Target/AArch64/AArch64StorePairSuppress.cpp index d5643d384283..0e84a00df006 100644 --- a/lib/Target/AArch64/AArch64StorePairSuppress.cpp +++ b/lib/Target/AArch64/AArch64StorePairSuppress.cpp @@ -1,9 +1,8 @@ //===--- AArch64StorePairSuppress.cpp --- Suppress store pair formation ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -148,7 +147,7 @@ bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) { for (auto &MI : MBB) { if (!isNarrowFPStore(MI)) continue; - MachineOperand *BaseOp; + const MachineOperand *BaseOp; int64_t Offset; if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI) && BaseOp->isReg()) { diff --git a/lib/Target/AArch64/AArch64Subtarget.cpp b/lib/Target/AArch64/AArch64Subtarget.cpp index dd30d25b2b50..3bc89b91c3f7 100644 --- a/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/lib/Target/AArch64/AArch64Subtarget.cpp @@ -1,9 +1,8 @@ //===-- AArch64Subtarget.cpp - AArch64 Subtarget Information ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -83,6 +82,7 @@ void AArch64Subtarget::initializeProperties() { case CortexA72: case CortexA73: case CortexA75: + case CortexA76: PrefFunctionAlignment = 4; break; case Cyclone: diff --git a/lib/Target/AArch64/AArch64Subtarget.h b/lib/Target/AArch64/AArch64Subtarget.h index 82f7bb755951..0c84cfb8329a 100644 --- a/lib/Target/AArch64/AArch64Subtarget.h +++ b/lib/Target/AArch64/AArch64Subtarget.h @@ -1,9 +1,8 @@ //===--- AArch64Subtarget.h - Define Subtarget for the AArch64 -*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,6 +45,7 @@ public: CortexA72, CortexA73, CortexA75, + CortexA76, Cyclone, ExynosM1, ExynosM3, @@ -93,6 +93,12 @@ protected: bool HasPAN_RWV = false; bool HasCCPP = false; + // Armv8.2 Crypto extensions + bool HasSM4 = false; + bool HasSHA3 = false; + bool HasSHA2 = false; + bool HasAES = false; + // ARMv8.3 extensions bool HasPA = false; bool HasJS = false; @@ -110,15 +116,10 @@ protected: bool HasTLB_RMI = false; bool HasFMI = false; bool HasRCPC_IMMO = false; - // ARMv8.4 Crypto extensions - bool HasSM4 = true; - bool HasSHA3 = true; - - bool HasSHA2 = true; - bool HasAES = true; bool HasLSLFast = false; bool HasSVE = false; + bool HasSVE2 = false; bool HasRCPC = false; bool HasAggressiveFMA = false; @@ -134,6 +135,12 @@ protected: bool HasRandGen = false; bool HasMTE = false; + // Arm SVE2 extensions + bool HasSVE2AES = false; + bool HasSVE2SM4 = false; + bool HasSVE2SHA3 = false; + bool HasSVE2BitPerm = false; + // HasZeroCycleRegMove - Has zero-cycle register mov instructions. bool HasZeroCycleRegMove = false; @@ -173,6 +180,9 @@ protected: bool DisableLatencySchedHeuristic = false; bool UseRSqrt = false; bool Force32BitJumpTables = false; + bool UseEL1ForTP = false; + bool UseEL2ForTP = false; + bool UseEL3ForTP = false; uint8_t MaxInterleaveFactor = 2; uint8_t VectorInsertExtractBaseCost = 3; uint16_t CacheLineSize = 0; @@ -324,6 +334,10 @@ public: hasFuseCCSelect() || hasFuseLiterals(); } + bool useEL1ForTP() const { return UseEL1ForTP; } + bool useEL2ForTP() const { return UseEL2ForTP; } + bool useEL3ForTP() const { return UseEL3ForTP; } + bool useRSqrt() const { return UseRSqrt; } bool force32BitJumpTables() const { return Force32BitJumpTables; } unsigned getMaxInterleaveFactor() const { return MaxInterleaveFactor; } @@ -353,6 +367,7 @@ public: bool hasSPE() const { return HasSPE; } bool hasLSLFast() const { return HasLSLFast; } bool hasSVE() const { return HasSVE; } + bool hasSVE2() const { return HasSVE2; } bool hasRCPC() const { return HasRCPC; } bool hasAggressiveFMA() const { return HasAggressiveFMA; } bool hasAlternativeNZCV() const { return HasAlternativeNZCV; } @@ -365,6 +380,11 @@ public: bool hasBTI() const { return HasBTI; } bool hasRandGen() const { return HasRandGen; } bool hasMTE() const { return HasMTE; } + // Arm SVE2 extensions + bool hasSVE2AES() const { return HasSVE2AES; } + bool hasSVE2SM4() const { return HasSVE2SM4; } + bool hasSVE2SHA3() const { return HasSVE2SHA3; } + bool hasSVE2BitPerm() const { return HasSVE2BitPerm; } bool isLittleEndian() const { return IsLittle; } diff --git a/lib/Target/AArch64/AArch64SystemOperands.td b/lib/Target/AArch64/AArch64SystemOperands.td index a804fb11175b..536a6591478b 100644 --- a/lib/Target/AArch64/AArch64SystemOperands.td +++ b/lib/Target/AArch64/AArch64SystemOperands.td @@ -1,9 +1,8 @@ //===- AArch64SystemOperands.td ----------------------------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1458,6 +1457,7 @@ def : RWSysReg<"TFSR_EL2", 0b11, 0b100, 0b0110, 0b0101, 0b000>; def : RWSysReg<"TFSR_EL3", 0b11, 0b110, 0b0110, 0b0110, 0b000>; def : RWSysReg<"TFSR_EL12", 0b11, 0b101, 0b0110, 0b0110, 0b000>; def : RWSysReg<"TFSRE0_EL1", 0b11, 0b000, 0b0110, 0b0110, 0b001>; +def : ROSysReg<"GMID_EL1", 0b11, 0b001, 0b0000, 0b0000, 0b100>; } // HasMTE // Cyclone specific system registers diff --git a/lib/Target/AArch64/AArch64TargetMachine.cpp b/lib/Target/AArch64/AArch64TargetMachine.cpp index 4e016525f7e4..865461480499 100644 --- a/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -1,9 +1,8 @@ //===-- AArch64TargetMachine.cpp - Define TargetMachine for AArch64 -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,9 +16,11 @@ #include "AArch64TargetObjectFile.h" #include "AArch64TargetTransformInfo.h" #include "MCTargetDesc/AArch64MCTargetDesc.h" +#include "TargetInfo/AArch64TargetInfo.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/CSEConfigBase.h" #include "llvm/CodeGen/GlobalISel/IRTranslator.h" #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" #include "llvm/CodeGen/GlobalISel/Legalizer.h" @@ -178,6 +179,7 @@ extern "C" void LLVMInitializeAArch64Target() { initializeFalkorMarkStridedAccessesLegacyPass(*PR); initializeLDTLSCleanupPass(*PR); initializeAArch64SpeculationHardeningPass(*PR); + initializeAArch64StackTaggingPass(*PR); } //===----------------------------------------------------------------------===// @@ -209,8 +211,8 @@ static std::string computeDataLayout(const Triple &TT, static Reloc::Model getEffectiveRelocModel(const Triple &TT, Optional<Reloc::Model> RM) { - // AArch64 Darwin is always PIC. - if (TT.isOSDarwin()) + // AArch64 Darwin and Windows are always PIC. + if (TT.isOSDarwin() || TT.isOSWindows()) return Reloc::PIC_; // On ELF platforms the default static relocation model has a smart enough // linker to cope with referencing external symbols defined in a shared @@ -384,6 +386,8 @@ public: void addPostRegAlloc() override; void addPreSched2() override; void addPreEmitPass() override; + + std::unique_ptr<CSEConfigBase> getCSEConfig() const override; }; } // end anonymous namespace @@ -397,6 +401,10 @@ TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) { return new AArch64PassConfig(*this, PM); } +std::unique_ptr<CSEConfigBase> AArch64PassConfig::getCSEConfig() const { + return getStandardCSEConfigForOpt(TM->getOptLevel()); +} + void AArch64PassConfig::addIRPasses() { // Always expand atomic operations, we don't deal with atomicrmw or cmpxchg // ourselves. @@ -439,6 +447,8 @@ void AArch64PassConfig::addIRPasses() { // invariant. addPass(createLICMPass()); } + + addPass(createAArch64StackTaggingPass()); } // Pass Pipeline Configuration @@ -455,7 +465,20 @@ bool AArch64PassConfig::addPreISel() { EnableGlobalMerge == cl::BOU_TRUE) { bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) && (EnableGlobalMerge == cl::BOU_UNSET); - addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize)); + + // Merging of extern globals is enabled by default on non-Mach-O as we + // expect it to be generally either beneficial or harmless. On Mach-O it + // is disabled as we emit the .subsections_via_symbols directive which + // means that merging extern globals is not safe. + bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO(); + + // FIXME: extern global merging is only enabled when we optimise for size + // because there are some regressions with it also enabled for performance. + if (!OnlyOptimizeForSize) + MergeExternalByDefault = false; + + addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize, + MergeExternalByDefault)); } return false; diff --git a/lib/Target/AArch64/AArch64TargetMachine.h b/lib/Target/AArch64/AArch64TargetMachine.h index 8d28a5e30ebf..5264efb89b9c 100644 --- a/lib/Target/AArch64/AArch64TargetMachine.h +++ b/lib/Target/AArch64/AArch64TargetMachine.h @@ -1,9 +1,8 @@ //==-- AArch64TargetMachine.h - Define TargetMachine for AArch64 -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/lib/Target/AArch64/AArch64TargetObjectFile.cpp index 8ae72a7ddb57..1c3d5d0743ad 100644 --- a/lib/Target/AArch64/AArch64TargetObjectFile.cpp +++ b/lib/Target/AArch64/AArch64TargetObjectFile.cpp @@ -1,9 +1,8 @@ //===-- AArch64TargetObjectFile.cpp - AArch64 Object Info -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64TargetObjectFile.h b/lib/Target/AArch64/AArch64TargetObjectFile.h index 9077eb7902fd..7ead363d42fe 100644 --- a/lib/Target/AArch64/AArch64TargetObjectFile.h +++ b/lib/Target/AArch64/AArch64TargetObjectFile.h @@ -1,9 +1,8 @@ //===-- AArch64TargetObjectFile.h - AArch64 Object Info -*- C++ ---------*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index a256cb7c9215..a4b78f2a7d6b 100644 --- a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -1,12 +1,12 @@ //===-- AArch64TargetTransformInfo.cpp - AArch64 specific TTI -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// +#include "AArch64ExpandImm.h" #include "AArch64TargetTransformInfo.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "llvm/Analysis/LoopInfo.h" @@ -50,8 +50,9 @@ int AArch64TTIImpl::getIntImmCost(int64_t Val) { Val = ~Val; // Calculate how many moves we will need to materialize this constant. - unsigned LZ = countLeadingZeros((uint64_t)Val); - return (64 - LZ + 15) / 16; + SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn; + AArch64_IMM::expandMOVImm(Val, 64, Insn); + return Insn.size(); } /// Calculate the cost of materializing the given constant. @@ -665,7 +666,7 @@ int AArch64TTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, assert(Factor >= 2 && "Invalid interleave factor"); assert(isa<VectorType>(VecTy) && "Expect a vector type"); - if (!UseMaskForCond && !UseMaskForGaps && + if (!UseMaskForCond && !UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) { unsigned NumElts = VecTy->getVectorNumElements(); auto *SubVecTy = VectorType::get(VecTy->getScalarType(), NumElts / Factor); diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.h b/lib/Target/AArch64/AArch64TargetTransformInfo.h index 08c1a8924220..10c15a139b4c 100644 --- a/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -1,9 +1,8 @@ //===- AArch64TargetTransformInfo.h - AArch64 specific TTI ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -166,6 +165,10 @@ public: return false; } + unsigned getGISelRematGlobalCost() const { + return 2; + } + bool useReductionIntrinsic(unsigned Opcode, Type *Ty, TTI::ReductionFlags Flags) const; diff --git a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index 6cc9b67e4d27..f4c55d48d215 100644 --- a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -1,9 +1,8 @@ //==- AArch64AsmParser.cpp - Parse AArch64 assembly to MCInst instructions -==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,6 +10,7 @@ #include "MCTargetDesc/AArch64MCExpr.h" #include "MCTargetDesc/AArch64MCTargetDesc.h" #include "MCTargetDesc/AArch64TargetStreamer.h" +#include "TargetInfo/AArch64TargetInfo.h" #include "AArch64InstrInfo.h" #include "Utils/AArch64BaseInfo.h" #include "llvm/ADT/APFloat.h" @@ -242,11 +242,13 @@ public: if (S.getTargetStreamer() == nullptr) new AArch64TargetStreamer(S); - // Alias .hword/.word/xword to the target-independent .2byte/.4byte/.8byte - // directives as they have the same form and semantics: - /// ::= (.hword | .word | .xword ) [ expression (, expression)* ] + // Alias .hword/.word/.[dx]word to the target-independent + // .2byte/.4byte/.8byte directives as they have the same form and + // semantics: + /// ::= (.hword | .word | .dword | .xword ) [ expression (, expression)* ] Parser.addAliasForDirective(".hword", ".2byte"); Parser.addAliasForDirective(".word", ".4byte"); + Parser.addAliasForDirective(".dword", ".8byte"); Parser.addAliasForDirective(".xword", ".8byte"); // Initialize the set of available features. @@ -1079,8 +1081,7 @@ public: if (Kind != k_Register || Reg.Kind != RegKind::SVEPredicateVector) return DiagnosticPredicateTy::NoMatch; - if (isSVEVectorReg<Class>() && - (ElementWidth == 0 || Reg.ElementWidth == ElementWidth)) + if (isSVEVectorReg<Class>() && (Reg.ElementWidth == ElementWidth)) return DiagnosticPredicateTy::Match; return DiagnosticPredicateTy::NearMatch; @@ -1091,8 +1092,7 @@ public: if (Kind != k_Register || Reg.Kind != RegKind::SVEDataVector) return DiagnosticPredicateTy::NoMatch; - if (isSVEVectorReg<Class>() && - (ElementWidth == 0 || Reg.ElementWidth == ElementWidth)) + if (isSVEVectorReg<Class>() && Reg.ElementWidth == ElementWidth) return DiagnosticPredicateTy::Match; return DiagnosticPredicateTy::NearMatch; @@ -1272,9 +1272,11 @@ public: bool isExtend64() const { if (!isExtend()) return false; - // UXTX and SXTX require a 64-bit source register (the ExtendLSL64 class). + // Make sure the extend expects a 32-bit source register. AArch64_AM::ShiftExtendType ET = getShiftExtendType(); - return ET != AArch64_AM::UXTX && ET != AArch64_AM::SXTX; + return ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB || + ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH || + ET == AArch64_AM::UXTW || ET == AArch64_AM::SXTW; } bool isExtendLSL64() const { @@ -2473,7 +2475,7 @@ OperandMatchResultTy AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) { MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); - const MCExpr *Expr; + const MCExpr *Expr = nullptr; if (Parser.getTok().is(AsmToken::Hash)) { Parser.Lex(); // Eat hash token. @@ -2500,6 +2502,7 @@ AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) { } else if (DarwinRefKind != MCSymbolRefExpr::VK_PAGE && DarwinRefKind != MCSymbolRefExpr::VK_GOTPAGE && DarwinRefKind != MCSymbolRefExpr::VK_TLVPPAGE && + ELFRefKind != AArch64MCExpr::VK_ABS_PAGE_NC && ELFRefKind != AArch64MCExpr::VK_GOT_PAGE && ELFRefKind != AArch64MCExpr::VK_GOTTPREL_PAGE && ELFRefKind != AArch64MCExpr::VK_TLSDESC_PAGE) { @@ -2523,7 +2526,7 @@ AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) { OperandMatchResultTy AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) { SMLoc S = getLoc(); - const MCExpr *Expr; + const MCExpr *Expr = nullptr; // Leave anything with a bracket to the default for SVE if (getParser().getTok().is(AsmToken::LBrac)) @@ -2621,7 +2624,7 @@ AArch64AsmParser::tryParseImmWithOptionalShift(OperandVector &Operands) { // Operand should start from # or should be integer, emit error otherwise. return MatchOperand_NoMatch; - const MCExpr *Imm; + const MCExpr *Imm = nullptr; if (parseSymbolicImmVal(Imm)) return MatchOperand_ParseFail; else if (Parser.getTok().isNot(AsmToken::Comma)) { @@ -2660,7 +2663,7 @@ AArch64AsmParser::tryParseImmWithOptionalShift(OperandVector &Operands) { Parser.Lex(); // Eat the number // Just in case the optional lsl #0 is used for immediates other than zero. - if (ShiftAmount == 0 && Imm != 0) { + if (ShiftAmount == 0 && Imm != nullptr) { SMLoc E = Parser.getTok().getLoc(); Operands.push_back(AArch64Operand::CreateImm(Imm, S, E, getContext())); return MatchOperand_Success; @@ -2833,6 +2836,11 @@ static const struct Extension { {"pan-rwv", {AArch64::FeaturePAN_RWV}}, {"ccpp", {AArch64::FeatureCCPP}}, {"sve", {AArch64::FeatureSVE}}, + {"sve2", {AArch64::FeatureSVE2}}, + {"sve2-aes", {AArch64::FeatureSVE2AES}}, + {"sve2-sm4", {AArch64::FeatureSVE2SM4}}, + {"sve2-sha3", {AArch64::FeatureSVE2SHA3}}, + {"bitperm", {AArch64::FeatureSVE2BitPerm}}, // FIXME: Unsupported extensions {"pan", {}}, {"lor", {}}, @@ -3260,6 +3268,7 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) { .Case("dtprel_hi12", AArch64MCExpr::VK_DTPREL_HI12) .Case("dtprel_lo12", AArch64MCExpr::VK_DTPREL_LO12) .Case("dtprel_lo12_nc", AArch64MCExpr::VK_DTPREL_LO12_NC) + .Case("pg_hi21_nc", AArch64MCExpr::VK_ABS_PAGE_NC) .Case("tprel_g2", AArch64MCExpr::VK_TPREL_G2) .Case("tprel_g1", AArch64MCExpr::VK_TPREL_G1) .Case("tprel_g1_nc", AArch64MCExpr::VK_TPREL_G1_NC) @@ -4098,15 +4107,6 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, SMLoc &IDLoc, "unpredictable STXP instruction, status is also a source"); break; } - case AArch64::LDGV: { - unsigned Rt = Inst.getOperand(0).getReg(); - unsigned Rn = Inst.getOperand(1).getReg(); - if (RI->isSubRegisterEq(Rt, Rn)) { - return Error(Loc[0], - "unpredictable LDGV instruction, writeback register is also " - "the target register"); - } - } } @@ -4167,7 +4167,8 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, SMLoc &IDLoc, } } -static std::string AArch64MnemonicSpellCheck(StringRef S, uint64_t FBS, +static std::string AArch64MnemonicSpellCheck(StringRef S, + const FeatureBitset &FBS, unsigned VariantID = 0); bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode, @@ -4199,7 +4200,7 @@ bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode, return Error(Loc, "expected AArch64 condition code"); case Match_AddSubRegExtendSmall: return Error(Loc, - "expected '[su]xt[bhw]' or 'lsl' with optional integer in range [0, 4]"); + "expected '[su]xt[bhw]' with optional integer in range [0, 4]"); case Match_AddSubRegExtendLarge: return Error(Loc, "expected 'sxtx' 'uxtx' or 'lsl' with optional integer in range [0, 4]"); @@ -4442,7 +4443,7 @@ bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode, case Match_InvalidZPR64LSL64: return Error(Loc, "invalid shift/extend specified, expected 'z[0..31].d, lsl #3'"); case Match_InvalidZPR0: - return Error(Loc, "expected register without element width sufix"); + return Error(Loc, "expected register without element width suffix"); case Match_InvalidZPR8: case Match_InvalidZPR16: case Match_InvalidZPR32: @@ -4470,11 +4471,15 @@ bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode, case Match_InvalidSVEPredicateDReg: return Error(Loc, "invalid predicate register."); case Match_InvalidSVEPredicate3bAnyReg: + return Error(Loc, "invalid restricted predicate register, expected p0..p7 (without element suffix)"); case Match_InvalidSVEPredicate3bBReg: + return Error(Loc, "invalid restricted predicate register, expected p0.b..p7.b"); case Match_InvalidSVEPredicate3bHReg: + return Error(Loc, "invalid restricted predicate register, expected p0.h..p7.h"); case Match_InvalidSVEPredicate3bSReg: + return Error(Loc, "invalid restricted predicate register, expected p0.s..p7.s"); case Match_InvalidSVEPredicate3bDReg: - return Error(Loc, "restricted predicate has range [0, 7]."); + return Error(Loc, "invalid restricted predicate register, expected p0.d..p7.d"); case Match_InvalidSVEExactFPImmOperandHalfOne: return Error(Loc, "Invalid floating point constant, expected 0.5 or 1.0."); case Match_InvalidSVEExactFPImmOperandHalfTwo: @@ -4777,10 +4782,12 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, } MCInst Inst; + FeatureBitset MissingFeatures; // First try to match against the secondary set of tables containing the // short-form NEON instructions (e.g. "fadd.2s v0, v1, v2"). unsigned MatchResult = - MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm, 1); + MatchInstructionImpl(Operands, Inst, ErrorInfo, MissingFeatures, + MatchingInlineAsm, 1); // If that fails, try against the alternate table containing long-form NEON: // "fadd v0.2s, v1.2s, v2.2s" @@ -4789,9 +4796,11 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, // long-form match also fails. auto ShortFormNEONErrorInfo = ErrorInfo; auto ShortFormNEONMatchResult = MatchResult; + auto ShortFormNEONMissingFeatures = MissingFeatures; MatchResult = - MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm, 0); + MatchInstructionImpl(Operands, Inst, ErrorInfo, MissingFeatures, + MatchingInlineAsm, 0); // Now, both matches failed, and the long-form match failed on the mnemonic // suffix token operand. The short-form match failure is probably more @@ -4801,6 +4810,7 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, ((AArch64Operand &)*Operands[1]).isTokenSuffix()) { MatchResult = ShortFormNEONMatchResult; ErrorInfo = ShortFormNEONErrorInfo; + MissingFeatures = ShortFormNEONMissingFeatures; } } @@ -4819,17 +4829,15 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, return false; } case Match_MissingFeature: { - assert(ErrorInfo && "Unknown missing feature!"); + assert(MissingFeatures.any() && "Unknown missing feature!"); // Special case the error message for the very common case where only // a single subtarget feature is missing (neon, e.g.). std::string Msg = "instruction requires:"; - uint64_t Mask = 1; - for (unsigned i = 0; i < (sizeof(ErrorInfo)*8-1); ++i) { - if (ErrorInfo & Mask) { + for (unsigned i = 0, e = MissingFeatures.size(); i != e; ++i) { + if (MissingFeatures[i]) { Msg += " "; - Msg += getSubtargetFeatureName(ErrorInfo & Mask); + Msg += getSubtargetFeatureName(i); } - Mask <<= 1; } return Error(IDLoc, Msg); } @@ -5148,7 +5156,7 @@ bool AArch64AsmParser::parseDirectiveArch(SMLoc L) { FeatureBitset ToggleFeatures = EnableFeature ? (~Features & Extension.Features) : ( Features & Extension.Features); - uint64_t Features = + FeatureBitset Features = ComputeAvailableFeatures(STI.ToggleFeature(ToggleFeatures)); setAvailableFeatures(Features); break; @@ -5160,15 +5168,9 @@ bool AArch64AsmParser::parseDirectiveArch(SMLoc L) { /// parseDirectiveArchExtension /// ::= .arch_extension [no]feature bool AArch64AsmParser::parseDirectiveArchExtension(SMLoc L) { - MCAsmParser &Parser = getParser(); - - if (getLexer().isNot(AsmToken::Identifier)) - return Error(getLexer().getLoc(), "expected architecture extension name"); + SMLoc ExtLoc = getLoc(); - const AsmToken &Tok = Parser.getTok(); - StringRef Name = Tok.getString(); - SMLoc ExtLoc = Tok.getLoc(); - Lex(); + StringRef Name = getParser().parseStringToEndOfStatement().trim(); if (parseToken(AsmToken::EndOfStatement, "unexpected token in '.arch_extension' directive")) @@ -5192,7 +5194,7 @@ bool AArch64AsmParser::parseDirectiveArchExtension(SMLoc L) { FeatureBitset ToggleFeatures = EnableFeature ? (~Features & Extension.Features) : (Features & Extension.Features); - uint64_t Features = + FeatureBitset Features = ComputeAvailableFeatures(STI.ToggleFeature(ToggleFeatures)); setAvailableFeatures(Features); return false; @@ -5257,7 +5259,7 @@ bool AArch64AsmParser::parseDirectiveCPU(SMLoc L) { FeatureBitset ToggleFeatures = EnableFeature ? (~Features & Extension.Features) : ( Features & Extension.Features); - uint64_t Features = + FeatureBitset Features = ComputeAvailableFeatures(STI.ToggleFeature(ToggleFeatures)); setAvailableFeatures(Features); FoundExtension = true; @@ -5518,6 +5520,8 @@ extern "C" void LLVMInitializeAArch64AsmParser() { RegisterMCAsmParser<AArch64AsmParser> X(getTheAArch64leTarget()); RegisterMCAsmParser<AArch64AsmParser> Y(getTheAArch64beTarget()); RegisterMCAsmParser<AArch64AsmParser> Z(getTheARM64Target()); + RegisterMCAsmParser<AArch64AsmParser> W(getTheARM64_32Target()); + RegisterMCAsmParser<AArch64AsmParser> V(getTheAArch64_32Target()); } #define GET_REGISTER_MATCHER diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp index 4102f1eb5cc1..145ffef6f6f9 100644 --- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp +++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp @@ -1,9 +1,8 @@ //===- AArch64Disassembler.cpp - Disassembler for AArch64 -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "AArch64ExternalSymbolizer.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "MCTargetDesc/AArch64MCTargetDesc.h" +#include "TargetInfo/AArch64TargetInfo.h" #include "Utils/AArch64BaseInfo.h" #include "llvm-c/Disassembler.h" #include "llvm/MC/MCDisassembler/MCRelocationInfo.h" @@ -220,11 +220,6 @@ static DecodeStatus DecodeImm8OptLsl(MCInst &Inst, unsigned Imm, static DecodeStatus DecodeSVEIncDecImm(MCInst &Inst, unsigned Imm, uint64_t Addr, const void *Decoder); -static DecodeStatus DecodeLoadAllocTagArrayInstruction(MCInst &Inst, - uint32_t insn, - uint64_t address, - const void* Decoder); - static bool Check(DecodeStatus &Out, DecodeStatus In) { switch (In) { case MCDisassembler::Success: @@ -292,11 +287,19 @@ extern "C" void LLVMInitializeAArch64Disassembler() { createAArch64ExternalSymbolizer); TargetRegistry::RegisterMCSymbolizer(getTheAArch64beTarget(), createAArch64ExternalSymbolizer); + TargetRegistry::RegisterMCDisassembler(getTheAArch64_32Target(), + createAArch64Disassembler); + TargetRegistry::RegisterMCSymbolizer(getTheAArch64_32Target(), + createAArch64ExternalSymbolizer); TargetRegistry::RegisterMCDisassembler(getTheARM64Target(), createAArch64Disassembler); TargetRegistry::RegisterMCSymbolizer(getTheARM64Target(), createAArch64ExternalSymbolizer); + TargetRegistry::RegisterMCDisassembler(getTheARM64_32Target(), + createAArch64Disassembler); + TargetRegistry::RegisterMCSymbolizer(getTheARM64_32Target(), + createAArch64ExternalSymbolizer); } static const unsigned FPR128DecoderTable[] = { @@ -1619,7 +1622,7 @@ static DecodeStatus DecodeModImmInstruction(MCInst &Inst, uint32_t insn, case AArch64::MOVIv4s_msl: case AArch64::MVNIv2s_msl: case AArch64::MVNIv4s_msl: - Inst.addOperand(MCOperand::createImm(cmode & 1 ? 0x110 : 0x108)); + Inst.addOperand(MCOperand::createImm((cmode & 1) ? 0x110 : 0x108)); break; } @@ -1779,8 +1782,8 @@ static DecodeStatus DecodeGPRSeqPairsClassRegisterClass(MCInst &Inst, if (RegNo & 0x1) return Fail; - unsigned Register = AArch64MCRegisterClasses[RegClassID].getRegister(RegNo); - Inst.addOperand(MCOperand::createReg(Register)); + unsigned Reg = AArch64MCRegisterClasses[RegClassID].getRegister(RegNo / 2); + Inst.addOperand(MCOperand::createReg(Reg)); return Success; } @@ -1852,25 +1855,3 @@ static DecodeStatus DecodeSVEIncDecImm(MCInst &Inst, unsigned Imm, Inst.addOperand(MCOperand::createImm(Imm + 1)); return Success; } - -static DecodeStatus DecodeLoadAllocTagArrayInstruction(MCInst &Inst, - uint32_t insn, - uint64_t address, - const void* Decoder) { - unsigned Rn = fieldFromInstruction(insn, 5, 5); - unsigned Rt = fieldFromInstruction(insn, 0, 5); - - // Outputs - DecodeGPR64spRegisterClass(Inst, Rn, address, Decoder); - DecodeGPR64RegisterClass(Inst, Rt, address, Decoder); - - // Input (Rn again) - Inst.addOperand(Inst.getOperand(0)); - - //Do this post decode since the raw number for xzr and sp is the same - if (Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) { - return SoftFail; - } else { - return Success; - } -} diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.h b/lib/Target/AArch64/Disassembler/AArch64Disassembler.h index bc2f7f181699..2ba5a695701f 100644 --- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.h +++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.h @@ -1,9 +1,8 @@ //===- AArch64Disassembler.h - Disassembler for AArch64 ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp index 342655a29b1d..3f815ac8c3d0 100644 --- a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp +++ b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp @@ -1,9 +1,8 @@ //===- AArch64ExternalSymbolizer.cpp - Symbolizer for AArch64 ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h index 49e844963797..dc72331660cc 100644 --- a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h +++ b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h @@ -1,9 +1,8 @@ //===- AArch64ExternalSymbolizer.h - Symbolizer for AArch64 -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h b/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h index 688ca755d0b5..05a909f1780a 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h @@ -1,9 +1,8 @@ //===- AArch64AddressingModes.h - AArch64 Addressing Modes ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index ed89d991d9fb..6418211a4f55 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -1,15 +1,15 @@ //===-- AArch64AsmBackend.cpp - AArch64 Assembler Backend -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include "AArch64.h" #include "MCTargetDesc/AArch64FixupKinds.h" #include "MCTargetDesc/AArch64MCExpr.h" +#include "MCTargetDesc/AArch64MCTargetDesc.h" +#include "Utils/AArch64BaseInfo.h" #include "llvm/ADT/Triple.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/MC/MCAsmBackend.h" @@ -22,8 +22,10 @@ #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCSectionMachO.h" +#include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/MCValue.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TargetRegistry.h" using namespace llvm; namespace { @@ -42,6 +44,8 @@ public: return AArch64::NumTargetFixupKinds; } + Optional<MCFixupKind> getFixupKind(StringRef Name) const override; + const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override { const static MCFixupKindInfo Infos[AArch64::NumTargetFixupKinds] = { // This table *must* be in the order that the fixup_* kinds are defined @@ -104,6 +108,7 @@ static unsigned getFixupKindNumBytes(unsigned Kind) { default: llvm_unreachable("Unknown fixup kind!"); + case FK_NONE: case AArch64::fixup_aarch64_tlsdesc_call: return 0; @@ -274,7 +279,7 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target, if (RefKind & AArch64MCExpr::VK_NC) { Value &= 0xFFFF; } - else if (RefKind & AArch64MCExpr::VK_SABS) { + else if (AArch64MCExpr::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) { if (SignedValue > 0xFFFF || SignedValue < -0xFFFF) Ctx.reportError(Fixup.getLoc(), "fixup value out of range"); @@ -305,6 +310,7 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target, if (Value & 0x3) Ctx.reportError(Fixup.getLoc(), "fixup not sufficiently aligned"); return (Value >> 2) & 0x3ffffff; + case FK_NONE: case FK_Data_1: case FK_Data_2: case FK_Data_4: @@ -315,6 +321,12 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target, } } +Optional<MCFixupKind> AArch64AsmBackend::getFixupKind(StringRef Name) const { + if (TheTriple.isOSBinFormatELF() && Name == "R_AARCH64_NONE") + return FK_NONE; + return MCAsmBackend::getFixupKind(Name); +} + /// getFixupKindContainereSizeInBytes - The number of bytes of the /// container involved in big endian or 0 if the item is little endian unsigned AArch64AsmBackend::getFixupKindContainereSizeInBytes(unsigned Kind) const { @@ -398,7 +410,7 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, // handle this more cleanly. This may affect the output of -show-mc-encoding. AArch64MCExpr::VariantKind RefKind = static_cast<AArch64MCExpr::VariantKind>(Target.getRefKind()); - if (RefKind & AArch64MCExpr::VK_SABS) { + if (AArch64MCExpr::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) { // If the immediate is negative, generate MOVN else MOVZ. // (Bit 30 = 0) ==> MOVN, (Bit 30 = 1) ==> MOVZ. if (SignedValue < 0) @@ -446,6 +458,10 @@ bool AArch64AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count) const { bool AArch64AsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target) { + unsigned Kind = Fixup.getKind(); + if (Kind == FK_NONE) + return true; + // The ADRP instruction adds some multiple of 0x1000 to the current PC & // ~0xfff. This means that the required offset to reach a symbol can vary by // up to one step depending on where the ADRP is in memory. For example: @@ -458,14 +474,14 @@ bool AArch64AsmBackend::shouldForceRelocation(const MCAssembler &Asm, // same page as the ADRP and the instruction should encode 0x0. Assuming the // section isn't 0x1000-aligned, we therefore need to delegate this decision // to the linker -- a relocation! - if ((uint32_t)Fixup.getKind() == AArch64::fixup_aarch64_pcrel_adrp_imm21) + if (Kind == AArch64::fixup_aarch64_pcrel_adrp_imm21) return true; AArch64MCExpr::VariantKind RefKind = static_cast<AArch64MCExpr::VariantKind>(Target.getRefKind()); AArch64MCExpr::VariantKind SymLoc = AArch64MCExpr::getSymbolLoc(RefKind); // LDR GOT relocations need a relocation - if ((uint32_t)Fixup.getKind() == AArch64::fixup_aarch64_ldr_pcrel_imm19 && + if (Kind == AArch64::fixup_aarch64_ldr_pcrel_imm19 && SymLoc == AArch64MCExpr::VK_GOT) return true; return false; @@ -513,6 +529,7 @@ enum CompactUnwindEncodings { // FIXME: This should be in a separate file. class DarwinAArch64AsmBackend : public AArch64AsmBackend { const MCRegisterInfo &MRI; + bool IsILP32; /// Encode compact unwind stack adjustment for frameless functions. /// See UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK in compact_unwind_encoding.h. @@ -523,13 +540,18 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend { public: DarwinAArch64AsmBackend(const Target &T, const Triple &TT, - const MCRegisterInfo &MRI) - : AArch64AsmBackend(T, TT, /*IsLittleEndian*/ true), MRI(MRI) {} + const MCRegisterInfo &MRI, bool IsILP32) + : AArch64AsmBackend(T, TT, /*IsLittleEndian*/ true), MRI(MRI), + IsILP32(IsILP32) {} std::unique_ptr<MCObjectTargetWriter> createObjectTargetWriter() const override { - return createAArch64MachObjectWriter(MachO::CPU_TYPE_ARM64, - MachO::CPU_SUBTYPE_ARM64_ALL); + if (IsILP32) + return createAArch64MachObjectWriter( + MachO::CPU_TYPE_ARM64_32, MachO::CPU_SUBTYPE_ARM64_32_V8, true); + else + return createAArch64MachObjectWriter(MachO::CPU_TYPE_ARM64, + MachO::CPU_SUBTYPE_ARM64_ALL, false); } /// Generate the compact unwind encoding from the CFI directives. @@ -711,8 +733,10 @@ MCAsmBackend *llvm::createAArch64leAsmBackend(const Target &T, const MCRegisterInfo &MRI, const MCTargetOptions &Options) { const Triple &TheTriple = STI.getTargetTriple(); - if (TheTriple.isOSBinFormatMachO()) - return new DarwinAArch64AsmBackend(T, TheTriple, MRI); + if (TheTriple.isOSBinFormatMachO()) { + const bool IsILP32 = TheTriple.isArch32Bit(); + return new DarwinAArch64AsmBackend(T, TheTriple, MRI, IsILP32); + } if (TheTriple.isOSBinFormatCOFF()) return new COFFAArch64AsmBackend(T, TheTriple); diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp index 2ccd7cef8bef..c871e2c62eac 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp @@ -1,9 +1,8 @@ //===-- AArch64ELFObjectWriter.cpp - AArch64 ELF Writer -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -186,6 +185,8 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx, if (IsILP32 && isNonILP32reloc(Fixup, RefKind, Ctx)) return ELF::R_AARCH64_NONE; switch ((unsigned)Fixup.getKind()) { + case FK_NONE: + return ELF::R_AARCH64_NONE; case FK_Data_1: Ctx.reportError(Fixup.getLoc(), "1-byte data relocations not supported"); return ELF::R_AARCH64_NONE; diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp index 9a7e34b0aeb1..c33f7e957b54 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp @@ -1,9 +1,8 @@ //===- lib/MC/AArch64ELFStreamer.cpp - ELF Object Output for AArch64 ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -103,8 +102,8 @@ public: /// This function is the one used to emit instruction data into the ELF /// streamer. We override it to add the appropriate mapping symbol if /// necessary. - void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI, - bool) override { + void EmitInstruction(const MCInst &Inst, + const MCSubtargetInfo &STI) override { EmitA64MappingSymbol(); MCELFStreamer::EmitInstruction(Inst, STI); } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h index d5b009ec30d1..25c609ee1496 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h @@ -1,9 +1,8 @@ //===-- AArch64ELFStreamer.h - ELF Streamer for AArch64 ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h b/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h index 4293dcba955e..fe8043fe5ec0 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h @@ -1,9 +1,8 @@ //===-- AArch64FixupKinds.h - AArch64 Specific Fixup Entries ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp index dcf2dd251149..d0a544273b8b 100644 --- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp @@ -1,9 +1,8 @@ //==-- AArch64InstPrinter.cpp - Convert AArch64 MCInst to assembly syntax --==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -293,6 +292,12 @@ void AArch64InstPrinter::printInst(const MCInst *MI, raw_ostream &O, printInstruction(MI, STI, O); printAnnotation(O, Annot); + + if (atomicBarrierDroppedOnZero(Opcode) && + (MI->getOperand(0).getReg() == AArch64::XZR || + MI->getOperand(0).getReg() == AArch64::WZR)) { + printAnnotation(O, "acquire semantics dropped since destination is zero"); + } } static bool isTblTbxInstruction(unsigned Opcode, StringRef &Layout, diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h b/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h index 4e9982f5b7be..5311f73ca21c 100644 --- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h @@ -1,9 +1,8 @@ //===-- AArch64InstPrinter.h - Convert AArch64 MCInst to assembly syntax --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -11,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_TARGET_AARCH64_INSTPRINTER_AARCH64INSTPRINTER_H -#define LLVM_LIB_TARGET_AARCH64_INSTPRINTER_AARCH64INSTPRINTER_H +#ifndef LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64INSTPRINTER_H +#define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64INSTPRINTER_H #include "MCTargetDesc/AArch64MCTargetDesc.h" #include "llvm/ADT/StringRef.h" @@ -220,4 +219,4 @@ public: } // end namespace llvm -#endif // LLVM_LIB_TARGET_AARCH64_INSTPRINTER_AARCH64INSTPRINTER_H +#endif // LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64INSTPRINTER_H diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp index 58e4a9c9a9e9..ecff1ab0a8b3 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp @@ -1,9 +1,8 @@ //===-- AArch64MCAsmInfo.cpp - AArch64 asm properties ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -131,8 +130,6 @@ AArch64MCAsmInfoGNUCOFF::AArch64MCAsmInfoGNUCOFF() { CodePointerSize = 8; CommentString = "//"; - ExceptionsType = ExceptionHandling::DwarfCFI; - // The default is dwarf, but WinEH can be enabled optionally, which requires - // WinEHEncodingType to be set. + ExceptionsType = ExceptionHandling::WinEH; WinEHEncodingType = WinEH::EncodingType::Itanium; } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h index e8570b1c2887..36ae92afc8c1 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h @@ -1,9 +1,8 @@ //=====-- AArch64MCAsmInfo.h - AArch64 asm properties ---------*- C++ -*--====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp index 41cad48f7aea..8cb7a1672983 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp @@ -1,9 +1,8 @@ //=- AArch64/AArch64MCCodeEmitter.cpp - Convert AArch64 code to machine code-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -188,9 +187,10 @@ public: const MCSubtargetInfo &STI) const; private: - uint64_t computeAvailableFeatures(const FeatureBitset &FB) const; - void verifyInstructionPredicates(const MCInst &MI, - uint64_t AvailableFeatures) const; + FeatureBitset computeAvailableFeatures(const FeatureBitset &FB) const; + void + verifyInstructionPredicates(const MCInst &MI, + const FeatureBitset &AvailableFeatures) const; }; } // end anonymous namespace diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp index 729486b1020c..0a529321edc8 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp @@ -1,9 +1,8 @@ //===-- AArch64MCExpr.cpp - AArch64 specific MC expression classes --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -80,8 +79,7 @@ StringRef AArch64MCExpr::getVariantKindName() const { } void AArch64MCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const { - if (getKind() != VK_NONE) - OS << getVariantKindName(); + OS << getVariantKindName(); Expr->print(OS, MAI); } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h index b6bf254d3835..ec9c95911628 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h @@ -1,9 +1,8 @@ //=--- AArch64MCExpr.h - AArch64 specific MC expression classes ---*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,8 +22,6 @@ namespace llvm { class AArch64MCExpr : public MCTargetExpr { public: enum VariantKind { - VK_NONE = 0x000, - // Symbol locations specifying (roughly speaking) what calculation should be // performed to construct the final address for the relocated // symbol. E.g. direct, via the GOT, ... diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp index 0f8198ba4e9b..df12274d9470 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp @@ -1,9 +1,8 @@ //===-- AArch64MCTargetDesc.cpp - AArch64 Target Descriptions ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,8 +14,10 @@ #include "AArch64ELFStreamer.h" #include "AArch64MCAsmInfo.h" #include "AArch64WinCOFFStreamer.h" -#include "InstPrinter/AArch64InstPrinter.h" #include "MCTargetDesc/AArch64AddressingModes.h" +#include "MCTargetDesc/AArch64InstPrinter.h" +#include "TargetInfo/AArch64TargetInfo.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCInstrAnalysis.h" @@ -56,11 +57,177 @@ createAArch64MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) { } void AArch64_MC::initLLVMToCVRegMapping(MCRegisterInfo *MRI) { - for (unsigned Reg = AArch64::NoRegister + 1; - Reg < AArch64::NUM_TARGET_REGS; ++Reg) { - unsigned CV = MRI->getEncodingValue(Reg); - MRI->mapLLVMRegToCVReg(Reg, CV); - } + // Mapping from CodeView to MC register id. + static const struct { + codeview::RegisterId CVReg; + MCPhysReg Reg; + } RegMap[] = { + {codeview::RegisterId::ARM64_W0, AArch64::W0}, + {codeview::RegisterId::ARM64_W1, AArch64::W1}, + {codeview::RegisterId::ARM64_W2, AArch64::W2}, + {codeview::RegisterId::ARM64_W3, AArch64::W3}, + {codeview::RegisterId::ARM64_W4, AArch64::W4}, + {codeview::RegisterId::ARM64_W5, AArch64::W5}, + {codeview::RegisterId::ARM64_W6, AArch64::W6}, + {codeview::RegisterId::ARM64_W7, AArch64::W7}, + {codeview::RegisterId::ARM64_W8, AArch64::W8}, + {codeview::RegisterId::ARM64_W9, AArch64::W9}, + {codeview::RegisterId::ARM64_W10, AArch64::W10}, + {codeview::RegisterId::ARM64_W11, AArch64::W11}, + {codeview::RegisterId::ARM64_W12, AArch64::W12}, + {codeview::RegisterId::ARM64_W13, AArch64::W13}, + {codeview::RegisterId::ARM64_W14, AArch64::W14}, + {codeview::RegisterId::ARM64_W15, AArch64::W15}, + {codeview::RegisterId::ARM64_W16, AArch64::W16}, + {codeview::RegisterId::ARM64_W17, AArch64::W17}, + {codeview::RegisterId::ARM64_W18, AArch64::W18}, + {codeview::RegisterId::ARM64_W19, AArch64::W19}, + {codeview::RegisterId::ARM64_W20, AArch64::W20}, + {codeview::RegisterId::ARM64_W21, AArch64::W21}, + {codeview::RegisterId::ARM64_W22, AArch64::W22}, + {codeview::RegisterId::ARM64_W23, AArch64::W23}, + {codeview::RegisterId::ARM64_W24, AArch64::W24}, + {codeview::RegisterId::ARM64_W25, AArch64::W25}, + {codeview::RegisterId::ARM64_W26, AArch64::W26}, + {codeview::RegisterId::ARM64_W27, AArch64::W27}, + {codeview::RegisterId::ARM64_W28, AArch64::W28}, + {codeview::RegisterId::ARM64_W29, AArch64::W29}, + {codeview::RegisterId::ARM64_W30, AArch64::W30}, + {codeview::RegisterId::ARM64_WZR, AArch64::WZR}, + {codeview::RegisterId::ARM64_X0, AArch64::X0}, + {codeview::RegisterId::ARM64_X1, AArch64::X1}, + {codeview::RegisterId::ARM64_X2, AArch64::X2}, + {codeview::RegisterId::ARM64_X3, AArch64::X3}, + {codeview::RegisterId::ARM64_X4, AArch64::X4}, + {codeview::RegisterId::ARM64_X5, AArch64::X5}, + {codeview::RegisterId::ARM64_X6, AArch64::X6}, + {codeview::RegisterId::ARM64_X7, AArch64::X7}, + {codeview::RegisterId::ARM64_X8, AArch64::X8}, + {codeview::RegisterId::ARM64_X9, AArch64::X9}, + {codeview::RegisterId::ARM64_X10, AArch64::X10}, + {codeview::RegisterId::ARM64_X11, AArch64::X11}, + {codeview::RegisterId::ARM64_X12, AArch64::X12}, + {codeview::RegisterId::ARM64_X13, AArch64::X13}, + {codeview::RegisterId::ARM64_X14, AArch64::X14}, + {codeview::RegisterId::ARM64_X15, AArch64::X15}, + {codeview::RegisterId::ARM64_X16, AArch64::X16}, + {codeview::RegisterId::ARM64_X17, AArch64::X17}, + {codeview::RegisterId::ARM64_X18, AArch64::X18}, + {codeview::RegisterId::ARM64_X19, AArch64::X19}, + {codeview::RegisterId::ARM64_X20, AArch64::X20}, + {codeview::RegisterId::ARM64_X21, AArch64::X21}, + {codeview::RegisterId::ARM64_X22, AArch64::X22}, + {codeview::RegisterId::ARM64_X23, AArch64::X23}, + {codeview::RegisterId::ARM64_X24, AArch64::X24}, + {codeview::RegisterId::ARM64_X25, AArch64::X25}, + {codeview::RegisterId::ARM64_X26, AArch64::X26}, + {codeview::RegisterId::ARM64_X27, AArch64::X27}, + {codeview::RegisterId::ARM64_X28, AArch64::X28}, + {codeview::RegisterId::ARM64_FP, AArch64::FP}, + {codeview::RegisterId::ARM64_LR, AArch64::LR}, + {codeview::RegisterId::ARM64_SP, AArch64::SP}, + {codeview::RegisterId::ARM64_ZR, AArch64::XZR}, + {codeview::RegisterId::ARM64_NZCV, AArch64::NZCV}, + {codeview::RegisterId::ARM64_S0, AArch64::S0}, + {codeview::RegisterId::ARM64_S1, AArch64::S1}, + {codeview::RegisterId::ARM64_S2, AArch64::S2}, + {codeview::RegisterId::ARM64_S3, AArch64::S3}, + {codeview::RegisterId::ARM64_S4, AArch64::S4}, + {codeview::RegisterId::ARM64_S5, AArch64::S5}, + {codeview::RegisterId::ARM64_S6, AArch64::S6}, + {codeview::RegisterId::ARM64_S7, AArch64::S7}, + {codeview::RegisterId::ARM64_S8, AArch64::S8}, + {codeview::RegisterId::ARM64_S9, AArch64::S9}, + {codeview::RegisterId::ARM64_S10, AArch64::S10}, + {codeview::RegisterId::ARM64_S11, AArch64::S11}, + {codeview::RegisterId::ARM64_S12, AArch64::S12}, + {codeview::RegisterId::ARM64_S13, AArch64::S13}, + {codeview::RegisterId::ARM64_S14, AArch64::S14}, + {codeview::RegisterId::ARM64_S15, AArch64::S15}, + {codeview::RegisterId::ARM64_S16, AArch64::S16}, + {codeview::RegisterId::ARM64_S17, AArch64::S17}, + {codeview::RegisterId::ARM64_S18, AArch64::S18}, + {codeview::RegisterId::ARM64_S19, AArch64::S19}, + {codeview::RegisterId::ARM64_S20, AArch64::S20}, + {codeview::RegisterId::ARM64_S21, AArch64::S21}, + {codeview::RegisterId::ARM64_S22, AArch64::S22}, + {codeview::RegisterId::ARM64_S23, AArch64::S23}, + {codeview::RegisterId::ARM64_S24, AArch64::S24}, + {codeview::RegisterId::ARM64_S25, AArch64::S25}, + {codeview::RegisterId::ARM64_S26, AArch64::S26}, + {codeview::RegisterId::ARM64_S27, AArch64::S27}, + {codeview::RegisterId::ARM64_S28, AArch64::S28}, + {codeview::RegisterId::ARM64_S29, AArch64::S29}, + {codeview::RegisterId::ARM64_S30, AArch64::S30}, + {codeview::RegisterId::ARM64_S31, AArch64::S31}, + {codeview::RegisterId::ARM64_D0, AArch64::D0}, + {codeview::RegisterId::ARM64_D1, AArch64::D1}, + {codeview::RegisterId::ARM64_D2, AArch64::D2}, + {codeview::RegisterId::ARM64_D3, AArch64::D3}, + {codeview::RegisterId::ARM64_D4, AArch64::D4}, + {codeview::RegisterId::ARM64_D5, AArch64::D5}, + {codeview::RegisterId::ARM64_D6, AArch64::D6}, + {codeview::RegisterId::ARM64_D7, AArch64::D7}, + {codeview::RegisterId::ARM64_D8, AArch64::D8}, + {codeview::RegisterId::ARM64_D9, AArch64::D9}, + {codeview::RegisterId::ARM64_D10, AArch64::D10}, + {codeview::RegisterId::ARM64_D11, AArch64::D11}, + {codeview::RegisterId::ARM64_D12, AArch64::D12}, + {codeview::RegisterId::ARM64_D13, AArch64::D13}, + {codeview::RegisterId::ARM64_D14, AArch64::D14}, + {codeview::RegisterId::ARM64_D15, AArch64::D15}, + {codeview::RegisterId::ARM64_D16, AArch64::D16}, + {codeview::RegisterId::ARM64_D17, AArch64::D17}, + {codeview::RegisterId::ARM64_D18, AArch64::D18}, + {codeview::RegisterId::ARM64_D19, AArch64::D19}, + {codeview::RegisterId::ARM64_D20, AArch64::D20}, + {codeview::RegisterId::ARM64_D21, AArch64::D21}, + {codeview::RegisterId::ARM64_D22, AArch64::D22}, + {codeview::RegisterId::ARM64_D23, AArch64::D23}, + {codeview::RegisterId::ARM64_D24, AArch64::D24}, + {codeview::RegisterId::ARM64_D25, AArch64::D25}, + {codeview::RegisterId::ARM64_D26, AArch64::D26}, + {codeview::RegisterId::ARM64_D27, AArch64::D27}, + {codeview::RegisterId::ARM64_D28, AArch64::D28}, + {codeview::RegisterId::ARM64_D29, AArch64::D29}, + {codeview::RegisterId::ARM64_D30, AArch64::D30}, + {codeview::RegisterId::ARM64_D31, AArch64::D31}, + {codeview::RegisterId::ARM64_Q0, AArch64::Q0}, + {codeview::RegisterId::ARM64_Q1, AArch64::Q1}, + {codeview::RegisterId::ARM64_Q2, AArch64::Q2}, + {codeview::RegisterId::ARM64_Q3, AArch64::Q3}, + {codeview::RegisterId::ARM64_Q4, AArch64::Q4}, + {codeview::RegisterId::ARM64_Q5, AArch64::Q5}, + {codeview::RegisterId::ARM64_Q6, AArch64::Q6}, + {codeview::RegisterId::ARM64_Q7, AArch64::Q7}, + {codeview::RegisterId::ARM64_Q8, AArch64::Q8}, + {codeview::RegisterId::ARM64_Q9, AArch64::Q9}, + {codeview::RegisterId::ARM64_Q10, AArch64::Q10}, + {codeview::RegisterId::ARM64_Q11, AArch64::Q11}, + {codeview::RegisterId::ARM64_Q12, AArch64::Q12}, + {codeview::RegisterId::ARM64_Q13, AArch64::Q13}, + {codeview::RegisterId::ARM64_Q14, AArch64::Q14}, + {codeview::RegisterId::ARM64_Q15, AArch64::Q15}, + {codeview::RegisterId::ARM64_Q16, AArch64::Q16}, + {codeview::RegisterId::ARM64_Q17, AArch64::Q17}, + {codeview::RegisterId::ARM64_Q18, AArch64::Q18}, + {codeview::RegisterId::ARM64_Q19, AArch64::Q19}, + {codeview::RegisterId::ARM64_Q20, AArch64::Q20}, + {codeview::RegisterId::ARM64_Q21, AArch64::Q21}, + {codeview::RegisterId::ARM64_Q22, AArch64::Q22}, + {codeview::RegisterId::ARM64_Q23, AArch64::Q23}, + {codeview::RegisterId::ARM64_Q24, AArch64::Q24}, + {codeview::RegisterId::ARM64_Q25, AArch64::Q25}, + {codeview::RegisterId::ARM64_Q26, AArch64::Q26}, + {codeview::RegisterId::ARM64_Q27, AArch64::Q27}, + {codeview::RegisterId::ARM64_Q28, AArch64::Q28}, + {codeview::RegisterId::ARM64_Q29, AArch64::Q29}, + {codeview::RegisterId::ARM64_Q30, AArch64::Q30}, + {codeview::RegisterId::ARM64_Q31, AArch64::Q31}, + + }; + for (unsigned I = 0; I < array_lengthof(RegMap); ++I) + MRI->mapLLVMRegToCVReg(RegMap[I].Reg, static_cast<int>(RegMap[I].CVReg)); } static MCRegisterInfo *createAArch64MCRegisterInfo(const Triple &Triple) { @@ -166,12 +333,20 @@ public: for (uint64_t Byte = 0, End = PltContents.size(); Byte + 7 < End; Byte += 4) { uint32_t Insn = support::endian::read32le(PltContents.data() + Byte); + uint64_t Off = 0; + // Check for optional bti c that prefixes adrp in BTI enabled entries + if (Insn == 0xd503245f) { + Off = 4; + Insn = support::endian::read32le(PltContents.data() + Byte + Off); + } // Check for adrp. if ((Insn & 0x9f000000) != 0x90000000) continue; + Off += 4; uint64_t Imm = (((PltSectionVA + Byte) >> 12) << 12) + (((Insn >> 29) & 3) << 12) + (((Insn >> 5) & 0x3ffff) << 14); - uint32_t Insn2 = support::endian::read32le(PltContents.data() + Byte + 4); + uint32_t Insn2 = + support::endian::read32le(PltContents.data() + Byte + Off); // Check for: ldr Xt, [Xn, #pimm]. if (Insn2 >> 22 == 0x3e5) { Imm += ((Insn2 >> 10) & 0xfff) << 3; @@ -192,7 +367,8 @@ static MCInstrAnalysis *createAArch64InstrAnalysis(const MCInstrInfo *Info) { // Force static initialization. extern "C" void LLVMInitializeAArch64TargetMC() { for (Target *T : {&getTheAArch64leTarget(), &getTheAArch64beTarget(), - &getTheARM64Target()}) { + &getTheAArch64_32Target(), &getTheARM64Target(), + &getTheARM64_32Target()}) { // Register the MC asm info. RegisterMCAsmInfoFn X(*T, createAArch64MCAsmInfo); @@ -228,7 +404,8 @@ extern "C" void LLVMInitializeAArch64TargetMC() { } // Register the asm backend. - for (Target *T : {&getTheAArch64leTarget(), &getTheARM64Target()}) + for (Target *T : {&getTheAArch64leTarget(), &getTheAArch64_32Target(), + &getTheARM64Target(), &getTheARM64_32Target()}) TargetRegistry::RegisterMCAsmBackend(*T, createAArch64leAsmBackend); TargetRegistry::RegisterMCAsmBackend(getTheAArch64beTarget(), createAArch64beAsmBackend); diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h index 0f22f69bd5b0..c84c313c1db0 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h @@ -1,9 +1,8 @@ //===-- AArch64MCTargetDesc.h - AArch64 Target Descriptions -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -37,10 +36,6 @@ class Triple; class raw_ostream; class raw_pwrite_stream; -Target &getTheAArch64leTarget(); -Target &getTheAArch64beTarget(); -Target &getTheARM64Target(); - MCCodeEmitter *createAArch64MCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI, MCContext &Ctx); @@ -57,7 +52,8 @@ std::unique_ptr<MCObjectTargetWriter> createAArch64ELFObjectWriter(uint8_t OSABI, bool IsILP32); std::unique_ptr<MCObjectTargetWriter> -createAArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype); +createAArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype, + bool IsILP32); std::unique_ptr<MCObjectTargetWriter> createAArch64WinCOFFObjectWriter(); diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp index 1021cdeeb3be..b3ce5ef22eef 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp @@ -1,9 +1,8 @@ //===-- AArch64MachObjectWriter.cpp - ARM Mach Object Writer --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -38,8 +37,8 @@ class AArch64MachObjectWriter : public MCMachObjectTargetWriter { unsigned &Log2Size, const MCAssembler &Asm); public: - AArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype) - : MCMachObjectTargetWriter(true /* is64Bit */, CPUType, CPUSubtype) {} + AArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype, bool IsILP32) + : MCMachObjectTargetWriter(!IsILP32 /* is64Bit */, CPUType, CPUSubtype) {} void recordRelocation(MachObjectWriter *Writer, MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, @@ -405,6 +404,8 @@ void AArch64MachObjectWriter::recordRelocation( } std::unique_ptr<MCObjectTargetWriter> -llvm::createAArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype) { - return llvm::make_unique<AArch64MachObjectWriter>(CPUType, CPUSubtype); +llvm::createAArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype, + bool IsILP32) { + return llvm::make_unique<AArch64MachObjectWriter>(CPUType, CPUSubtype, + IsILP32); } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp index a6b8d963bef9..f70752f5303f 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp @@ -1,9 +1,8 @@ //===- AArch64TargetStreamer.cpp - AArch64TargetStreamer class ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,6 +12,7 @@ #include "AArch64TargetStreamer.h" #include "llvm/MC/ConstantPools.h" +#include "llvm/MC/MCSection.h" #include "llvm/MC/MCSubtargetInfo.h" using namespace llvm; diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h index 73fb9baea3e3..3a0c5d8318dd 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h @@ -1,9 +1,8 @@ //===-- AArch64TargetStreamer.h - AArch64 Target Streamer ------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp index 7ea7d5f2a20e..a45880a07427 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp @@ -1,9 +1,8 @@ //= AArch64WinCOFFObjectWriter.cpp - AArch64 Windows COFF Object Writer C++ =// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp index b828ab832e9d..37c6fbb03908 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp @@ -1,9 +1,8 @@ //===-- AArch64WinCOFFStreamer.cpp - ARM Target WinCOFF Streamer ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h index ed265a876ab3..8c0656652eed 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h @@ -1,9 +1,8 @@ //===-- AArch64WinCOFFStreamer.h - WinCOFF Streamer for AArch64 -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/SVEInstrFormats.td b/lib/Target/AArch64/SVEInstrFormats.td index 23a65b345bad..808e59467081 100644 --- a/lib/Target/AArch64/SVEInstrFormats.td +++ b/lib/Target/AArch64/SVEInstrFormats.td @@ -1,9 +1,8 @@ //=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -701,8 +700,8 @@ multiclass sve_int_perm_dup_i<string asm> { (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; } -class sve_int_perm_tbl<bits<2> sz8_64, string asm, ZPRRegOp zprty, - RegisterOperand VecList> +class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, + ZPRRegOp zprty, RegisterOperand VecList> : I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), asm, "\t$Zd, $Zn, $Zm", "", @@ -714,16 +713,18 @@ class sve_int_perm_tbl<bits<2> sz8_64, string asm, ZPRRegOp zprty, let Inst{23-22} = sz8_64; let Inst{21} = 0b1; let Inst{20-16} = Zm; - let Inst{15-10} = 0b001100; + let Inst{15-13} = 0b001; + let Inst{12-11} = opc; + let Inst{10} = 0b0; let Inst{9-5} = Zn; let Inst{4-0} = Zd; } multiclass sve_int_perm_tbl<string asm> { - def _B : sve_int_perm_tbl<0b00, asm, ZPR8, Z_b>; - def _H : sve_int_perm_tbl<0b01, asm, ZPR16, Z_h>; - def _S : sve_int_perm_tbl<0b10, asm, ZPR32, Z_s>; - def _D : sve_int_perm_tbl<0b11, asm, ZPR64, Z_d>; + def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; + def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; + def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; + def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; def : InstAlias<asm # "\t$Zd, $Zn, $Zm", (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; @@ -735,6 +736,37 @@ multiclass sve_int_perm_tbl<string asm> { (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; } +multiclass sve2_int_perm_tbl<string asm> { + def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; + def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; + def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; + def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; +} + +class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> +: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), + asm, "\t$Zd, $Zn, $Zm", + "", + []>, Sched<[]> { + bits<5> Zd; + bits<5> Zm; + bits<5> Zn; + let Inst{31-24} = 0b00000101; + let Inst{23-22} = sz8_64; + let Inst{21} = 0b1; + let Inst{20-16} = Zm; + let Inst{15-10} = 0b001011; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_perm_tbx<string asm> { + def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; + def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; + def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; + def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; +} + class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> : I<(outs zprty:$Zd), (ins zprty:$Zn), asm, "\t$Zd, $Zn", @@ -875,6 +907,21 @@ class sve_int_perm_extract_i<string asm> let ElementSize = ElementSizeNone; } +class sve2_int_perm_extract_i_cons<string asm> +: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), + asm, "\t$Zd, $Zn, $imm8", + "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<8> imm8; + let Inst{31-21} = 0b00000101011; + let Inst{20-16} = imm8{7-3}; + let Inst{15-13} = 0b000; + let Inst{12-10} = imm8{2-0}; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + //===----------------------------------------------------------------------===// // SVE Vector Select Group //===----------------------------------------------------------------------===// @@ -1437,6 +1484,132 @@ multiclass sve_fp_fcadd<string asm> { } //===----------------------------------------------------------------------===// +// SVE2 Floating Point Convert Group +//===----------------------------------------------------------------------===// + +class sve2_fp_convert_precision<bits<4> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zd), (ins PPR3bAny:$Pg, zprty2:$Zn), + asm, "\t$Zd, $Pg/m, $Zn", + "", + []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<3> Pg; + let Inst{31-24} = 0b01100100; + let Inst{23-22} = opc{3-2}; + let Inst{21-18} = 0b0010; + let Inst{17-16} = opc{1-0}; + let Inst{15-13} = 0b101; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_fp_convert_down_narrow<string asm> { + def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; + def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; +} + +multiclass sve2_fp_convert_up_long<string asm> { + def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; + def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; +} + +multiclass sve2_fp_convert_down_odd_rounding<string asm> { + def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Floating Point Pairwise Group +//===----------------------------------------------------------------------===// + +class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, + ZPRRegOp zprty> +: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), + asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", + "", + []>, Sched<[]> { + bits<3> Pg; + bits<5> Zm; + bits<5> Zdn; + let Inst{31-24} = 0b01100100; + let Inst{23-22} = sz; + let Inst{21-19} = 0b010; + let Inst{18-16} = opc; + let Inst{15-13} = 0b100; + let Inst{12-10} = Pg; + let Inst{9-5} = Zm; + let Inst{4-0} = Zdn; + + let Constraints = "$Zdn = $_Zdn"; + let DestructiveInstType = Destructive; + let ElementSize = zprty.ElementSize; +} + +multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm> { + def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; + def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; + def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Floating Point Widening Multiply-Add - Indexed Group +//===----------------------------------------------------------------------===// + +class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> +: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, + VectorIndexH:$iop), + asm, "\t$Zda, $Zn, $Zm$iop", + "", + []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + bits<3> Zm; + bits<3> iop; + let Inst{31-21} = 0b01100100101; + let Inst{20-19} = iop{2-1}; + let Inst{18-16} = Zm; + let Inst{15-14} = 0b01; + let Inst{13} = opc{1}; + let Inst{12} = 0b0; + let Inst{11} = iop{0}; + let Inst{10} = opc{0}; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +//===----------------------------------------------------------------------===// +// SVE2 Floating Point Widening Multiply-Add Group +//===----------------------------------------------------------------------===// + +class sve2_fp_mla_long<bits<2> opc, string asm> +: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), + asm, "\t$Zda, $Zn, $Zm", + "", + []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + bits<5> Zm; + let Inst{31-21} = 0b01100100101; + let Inst{20-16} = Zm; + let Inst{15-14} = 0b10; + let Inst{13} = opc{1}; + let Inst{12-11} = 0b00; + let Inst{10} = opc{0}; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +//===----------------------------------------------------------------------===// // SVE Stack Allocation Group //===----------------------------------------------------------------------===// @@ -1536,6 +1709,12 @@ multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm> { def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>; } +multiclass sve2_fp_flogb<string asm> { + def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; + def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; + def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; +} + //===----------------------------------------------------------------------===// // SVE Floating Point Unary Operations - Unpredicated Group //===----------------------------------------------------------------------===// @@ -1692,6 +1871,112 @@ multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm> { } //===----------------------------------------------------------------------===// +// SVE2 Integer Multiply-Add - Unpredicated Group +//===----------------------------------------------------------------------===// + +class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), + asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + bits<5> Zm; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21} = 0b0; + let Inst{20-16} = Zm; + let Inst{15} = 0b0; + let Inst{14-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_int_mla<bit S, string asm> { + def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; + def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; + def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; + def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; +} + +multiclass sve2_int_mla_long<bits<5> opc, string asm> { + def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; + def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; + def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Integer Multiply-Add - Indexed Group +//===----------------------------------------------------------------------===// + +class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2, + ZPRRegOp zprty3, Operand itype> +: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), + asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21} = 0b1; + let Inst{15-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm> { + def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH> { + bits<3> Zm; + bits<3> iop; + let Inst{22} = iop{2}; + let Inst{20-19} = iop{1-0}; + let Inst{18-16} = Zm; + } + def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS> { + bits<3> Zm; + bits<2> iop; + let Inst{20-19} = iop; + let Inst{18-16} = Zm; + } + def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD> { + bits<4> Zm; + bit iop; + let Inst{20} = iop; + let Inst{19-16} = Zm; + } +} + +//===----------------------------------------------------------------------===// +// SVE2 Integer Multiply-Add Long - Indexed Group +//===----------------------------------------------------------------------===// + +multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm> { + def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, + asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH> { + bits<3> Zm; + bits<3> iop; + let Inst{20-19} = iop{2-1}; + let Inst{18-16} = Zm; + let Inst{11} = iop{0}; + } + def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, + asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS> { + bits<4> Zm; + bits<2> iop; + let Inst{20} = iop{1}; + let Inst{19-16} = Zm; + let Inst{11} = iop{0}; + } +} + +//===----------------------------------------------------------------------===// // SVE Integer Dot Product Group //===----------------------------------------------------------------------===// @@ -1762,6 +2047,645 @@ multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm> { } //===----------------------------------------------------------------------===// +// SVE2 Complex Integer Dot Product Group +//===----------------------------------------------------------------------===// + +class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, + complexrotateop:$rot), + asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + bits<5> Zm; + bits<2> rot; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21} = 0b0; + let Inst{20-16} = Zm; + let Inst{15-12} = opc; + let Inst{11-10} = rot; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_cintx_dot<string asm> { + def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; + def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Complex Multiply-Add Group +//===----------------------------------------------------------------------===// + +multiclass sve2_int_cmla<bit opc, string asm> { + def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; + def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; + def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; + def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Complex Integer Dot Product - Indexed Group +//===----------------------------------------------------------------------===// + +class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2, + ZPRRegOp zprty3, Operand itype> +: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, + complexrotateop:$rot), + asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + bits<2> rot; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21} = 0b1; + let Inst{15-12} = opc; + let Inst{11-10} = rot; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_cintx_dot_by_indexed_elem<string asm> { + def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS> { + bits<2> iop; + bits<3> Zm; + let Inst{20-19} = iop; + let Inst{18-16} = Zm; + } + def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD> { + bit iop; + bits<4> Zm; + let Inst{20} = iop; + let Inst{19-16} = Zm; + } +} + +//===----------------------------------------------------------------------===// +// SVE2 Complex Multiply-Add - Indexed Group +//===----------------------------------------------------------------------===// + +multiclass sve2_cmla_by_indexed_elem<bit opc, string asm> { + def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS> { + bits<2> iop; + bits<3> Zm; + let Inst{20-19} = iop; + let Inst{18-16} = Zm; + } + def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD> { + bit iop; + bits<4> Zm; + let Inst{20} = iop; + let Inst{19-16} = Zm; + } +} + +//===----------------------------------------------------------------------===// +// SVE2 Integer Multiply - Unpredicated Group +//===----------------------------------------------------------------------===// + +class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> +: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), + asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zm; + bits<5> Zn; + let Inst{31-24} = 0b00000100; + let Inst{23-22} = sz; + let Inst{21} = 0b1; + let Inst{20-16} = Zm; + let Inst{15-13} = 0b011; + let Inst{12-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_mul<bits<3> opc, string asm> { + def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; + def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; + def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; + def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Integer Multiply - Indexed Group +//===----------------------------------------------------------------------===// + +class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2, + ZPRRegOp zprty3, Operand itype> +: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), + asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21} = 0b1; + let Inst{15-14} = 0b11; + let Inst{13-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm> { + def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH> { + bits<3> Zm; + bits<3> iop; + let Inst{22} = iop{2}; + let Inst{20-19} = iop{1-0}; + let Inst{18-16} = Zm; + } + def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS> { + bits<3> Zm; + bits<2> iop; + let Inst{20-19} = iop; + let Inst{18-16} = Zm; + } + def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD> { + bits<4> Zm; + bit iop; + let Inst{20} = iop; + let Inst{19-16} = Zm; + } +} + +multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm> { + def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, + ZPR32, ZPR16, ZPR3b16, VectorIndexH> { + bits<3> Zm; + bits<3> iop; + let Inst{20-19} = iop{2-1}; + let Inst{18-16} = Zm; + let Inst{11} = iop{0}; + } + def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, + ZPR64, ZPR32, ZPR4b32, VectorIndexS> { + bits<4> Zm; + bits<2> iop; + let Inst{20} = iop{1}; + let Inst{19-16} = Zm; + let Inst{11} = iop{0}; + } +} + +//===----------------------------------------------------------------------===// +// SVE2 Integer - Predicated Group +//===----------------------------------------------------------------------===// + +class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, + ZPRRegOp zprty> +: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), + asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { + bits<3> Pg; + bits<5> Zm; + bits<5> Zdn; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21} = 0b0; + let Inst{20-16} = opc{5-1}; + let Inst{15-14} = 0b10; + let Inst{13} = opc{0}; + let Inst{12-10} = Pg; + let Inst{9-5} = Zm; + let Inst{4-0} = Zdn; + + let Constraints = "$Zdn = $_Zdn"; + let DestructiveInstType = Destructive; + let ElementSize = zprty.ElementSize; +} + +multiclass sve2_int_arith_pred<bits<6> opc, string asm> { + def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>; + def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>; + def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>; + def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>; +} + +class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), + asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { + bits<3> Pg; + bits<5> Zn; + bits<5> Zda; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21-17} = 0b00010; + let Inst{16} = U; + let Inst{15-13} = 0b101; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = zprty1.ElementSize; +} + +multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm> { + def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; + def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; + def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; +} + +class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, + string asm, ZPRRegOp zprty> +: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), + asm, "\t$Zd, $Pg/m, $Zn", + "", + []>, Sched<[]> { + bits<3> Pg; + bits<5> Zd; + bits<5> Zn; + let Inst{31-24} = 0b01000100; + let Inst{23-22} = sz; + let Inst{21-20} = 0b00; + let Inst{19} = Q; + let Inst{18} = 0b0; + let Inst{17-16} = opc; + let Inst{15-13} = 0b101; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; + + let Constraints = "$Zd = $_Zd"; + let DestructiveInstType = Destructive; + let ElementSize = zprty.ElementSize; +} + +multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm> { + def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>; +} + +multiclass sve2_int_un_pred_arit<bits<3> opc, string asm> { + def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>; + def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>; + def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>; + def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Widening Integer Arithmetic Group +//===----------------------------------------------------------------------===// + +class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> +: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), + asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<5> Zm; + let Inst{31-24} = 0b01000101; + let Inst{23-22} = sz; + let Inst{21} = 0b0; + let Inst{20-16} = Zm; + let Inst{15} = 0b0; + let Inst{14-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_wide_int_arith_long<bits<5> opc, string asm> { + def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; + def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; + def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; +} + +multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm> { + def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; + def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; + def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; +} + +multiclass sve2_pmul_long<bits<1> opc, string asm> { + def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; + def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Misc Group +//===----------------------------------------------------------------------===// + +class sve2_misc<bits<2> sz, bits<4> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), + asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<5> Zm; + let Inst{31-24} = 0b01000101; + let Inst{23-22} = sz; + let Inst{21} = 0b0; + let Inst{20-16} = Zm; + let Inst{15-14} = 0b10; + let Inst{13-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_misc_bitwise<bits<4> opc, string asm> { + def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; + def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; + def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; + def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; +} + +multiclass sve2_bitwise_xor_interleaved<bit opc, string asm> { + let DestructiveInstType = Destructive, ElementSize = ElementSizeNone in { + def _B : sve2_misc<0b00, { 0b010, opc }, asm, ZPR8, ZPR8>; + def _H : sve2_misc<0b01, { 0b010, opc }, asm, ZPR16, ZPR16>; + def _S : sve2_misc<0b10, { 0b010, opc }, asm, ZPR32, ZPR32>; + def _D : sve2_misc<0b11, { 0b010, opc }, asm, ZPR64, ZPR64>; + } +} + +multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm> { + def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; + def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; + def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; +} + +class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2, + Operand immtype> +: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), + asm, "\t$Zd, $Zn, $imm", + "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<5> imm; + let Inst{31-23} = 0b010001010; + let Inst{22} = tsz8_64{2}; + let Inst{21} = 0b0; + let Inst{20-19} = tsz8_64{1-0}; + let Inst{18-16} = imm{2-0}; // imm3 + let Inst{15-12} = 0b1010; + let Inst{11-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm> { + def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, + ZPR16, ZPR8, vecshiftL8>; + def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, + ZPR32, ZPR16, vecshiftL16> { + let Inst{19} = imm{3}; + } + def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, + ZPR64, ZPR32, vecshiftL32> { + let Inst{20-19} = imm{4-3}; + } +} + +//===----------------------------------------------------------------------===// +// SVE2 Accumulate Group +//===----------------------------------------------------------------------===// + +class sve2_int_bin_cons_shift_imm<bits<4> tsz8_64, bit opc, string asm, + ZPRRegOp zprty, Operand immtype> +: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), + asm, "\t$Zd, $Zn, $imm", + "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<6> imm; + let Inst{31-24} = 0b01000101; + let Inst{23-22} = tsz8_64{3-2}; + let Inst{21} = 0b0; + let Inst{20-19} = tsz8_64{1-0}; + let Inst{18-16} = imm{2-0}; // imm3 + let Inst{15-11} = 0b11110; + let Inst{10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_bin_cons_shift_imm_left<bit opc, string asm> { + def _B : sve2_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; + def _H : sve2_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { + let Inst{19} = imm{3}; + } + def _S : sve2_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { + let Inst{20-19} = imm{4-3}; + } + def _D : sve2_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { + let Inst{22} = imm{5}; + let Inst{20-19} = imm{4-3}; + } +} + +multiclass sve2_int_bin_cons_shift_imm_right<bit opc, string asm> { + def _B : sve2_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; + def _H : sve2_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { + let Inst{19} = imm{3}; + } + def _S : sve2_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { + let Inst{20-19} = imm{4-3}; + } + def _D : sve2_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { + let Inst{22} = imm{5}; + let Inst{20-19} = imm{4-3}; + } +} + +class sve2_int_bin_accum_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, + ZPRRegOp zprty, Operand immtype> +: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), + asm, "\t$Zda, $Zn, $imm", + "", []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + bits<6> imm; + let Inst{31-24} = 0b01000101; + let Inst{23-22} = tsz8_64{3-2}; + let Inst{21} = 0b0; + let Inst{20-19} = tsz8_64{1-0}; + let Inst{18-16} = imm{2-0}; // imm3 + let Inst{15-12} = 0b1110; + let Inst{11-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_int_bin_accum_cons_shift_imm_right<bits<2> opc, string asm> { + def _B : sve2_int_bin_accum_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; + def _H : sve2_int_bin_accum_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { + let Inst{19} = imm{3}; + } + def _S : sve2_int_bin_accum_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { + let Inst{20-19} = imm{4-3}; + } + def _D : sve2_int_bin_accum_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { + let Inst{22} = imm{5}; + let Inst{20-19} = imm{4-3}; + } +} + +class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> +: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), + asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { + bits<5> Zdn; + bits<5> Zm; + bit rot; + let Inst{31-24} = 0b01000101; + let Inst{23-22} = sz; + let Inst{21-17} = 0b00000; + let Inst{16} = opc; + let Inst{15-11} = 0b11011; + let Inst{10} = rot; + let Inst{9-5} = Zm; + let Inst{4-0} = Zdn; + + let Constraints = "$Zdn = $_Zdn"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_int_cadd<bit opc, string asm> { + def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; + def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; + def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; + def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; +} + +class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), + asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { + bits<5> Zda; + bits<5> Zn; + bits<5> Zm; + let Inst{31-24} = 0b01000101; + let Inst{23-22} = sz; + let Inst{21} = 0b0; + let Inst{20-16} = Zm; + let Inst{15-14} = 0b11; + let Inst{13-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zda; + + let Constraints = "$Zda = $_Zda"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_int_absdiff_accum<bit opc, string asm> { + def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; + def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; + def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; + def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; +} + +multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm> { + def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; + def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; + def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; +} + +multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm> { + def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, + ZPR32, ZPR32>; + def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, + ZPR64, ZPR64>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Narrowing Group +//===----------------------------------------------------------------------===// + +class sve2_int_bin_cons_shift_imm_narrow<bits<3> tsz8_64, bits<4> opc, + string asm, ZPRRegOp zprty1, + ZPRRegOp zprty2, Operand immtype> +: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), + asm, "\t$Zd, $Zn, $imm", + "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<5> imm; + let Inst{31-23} = 0b010001010; + let Inst{22} = tsz8_64{2}; + let Inst{21} = 0b1; + let Inst{20-19} = tsz8_64{1-0}; + let Inst{18-16} = imm{2-0}; // imm3 + let Inst{15-14} = 0b00; + let Inst{13-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_bin_cons_shift_imm_right_narrow<bits<4> opc, string asm> { + def _B : sve2_int_bin_cons_shift_imm_narrow<{0,0,1}, opc, asm, ZPR8, ZPR16, + vecshiftR8>; + def _H : sve2_int_bin_cons_shift_imm_narrow<{0,1,?}, opc, asm, ZPR16, ZPR32, + vecshiftR16> { + let Inst{19} = imm{3}; + } + def _S : sve2_int_bin_cons_shift_imm_narrow<{1,?,?}, opc, asm, ZPR32, ZPR64, + vecshiftR32> { + let Inst{20-19} = imm{4-3}; + } +} + +class sve2_int_addsub_narrow_high<bits<2> sz, bits<3> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), + asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<5> Zm; + let Inst{31-24} = 0b01000101; + let Inst{23-22} = sz; + let Inst{21} = 0b1; + let Inst{20-16} = Zm; + let Inst{15-13} = 0b011; + let Inst{12-10} = opc; // S, R, T + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_addsub_narrow_high<bits<3> opc, string asm> { + def _B : sve2_int_addsub_narrow_high<0b01, opc, asm, ZPR8, ZPR16>; + def _H : sve2_int_addsub_narrow_high<0b10, opc, asm, ZPR16, ZPR32>; + def _S : sve2_int_addsub_narrow_high<0b11, opc, asm, ZPR32, ZPR64>; +} + +class sve2_int_sat_extract_narrow<bits<3> tsz8_64, bits<3> opc, string asm, + ZPRRegOp zprty1, ZPRRegOp zprty2> +: I<(outs zprty1:$Zd), (ins zprty2:$Zn), + asm, "\t$Zd, $Zn", "", []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + let Inst{31-23} = 0b010001010; + let Inst{22} = tsz8_64{2}; + let Inst{21} = 0b1; + let Inst{20-19} = tsz8_64{1-0}; + let Inst{18-13} = 0b000010; + let Inst{12-10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_sat_extract_narrow<bits<3> opc, string asm> { + def _B : sve2_int_sat_extract_narrow<0b001, opc, asm, ZPR8, ZPR16>; + def _H : sve2_int_sat_extract_narrow<0b010, opc, asm, ZPR16, ZPR32>; + def _S : sve2_int_sat_extract_narrow<0b100, opc, asm, ZPR32, ZPR64>; +} + +//===----------------------------------------------------------------------===// // SVE Integer Arithmetic - Unary Predicated Group //===----------------------------------------------------------------------===// @@ -1983,6 +2907,86 @@ class sve_int_bin_cons_log<bits<2> opc, string asm> let Inst{4-0} = Zd; } +multiclass sve_int_bin_cons_log<bits<2> opc, string asm> { + def NAME : sve_int_bin_cons_log<opc, asm>; + + def : InstAlias<asm # "\t$Zd, $Zn, $Zm", + (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; + def : InstAlias<asm # "\t$Zd, $Zn, $Zm", + (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; + def : InstAlias<asm # "\t$Zd, $Zn, $Zm", + (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; +} + +class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> +: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), + asm, "\t$Zdn, $_Zdn, $Zm, $Zk", + "", + []>, Sched<[]> { + bits<5> Zdn; + bits<5> Zk; + bits<5> Zm; + let Inst{31-24} = 0b00000100; + let Inst{23-22} = opc{2-1}; + let Inst{21} = 0b1; + let Inst{20-16} = Zm; + let Inst{15-11} = 0b00111; + let Inst{10} = opc{0}; + let Inst{9-5} = Zk; + let Inst{4-0} = Zdn; + + let Constraints = "$Zdn = $_Zdn"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm> { + def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; + + def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", + (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; + def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", + (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; + def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", + (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; +} + +class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, + ZPRRegOp zprty, Operand immtype> +: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), + asm, "\t$Zdn, $_Zdn, $Zm, $imm", + "", + []>, Sched<[]> { + bits<5> Zdn; + bits<5> Zm; + bits<6> imm; + let Inst{31-24} = 0b00000100; + let Inst{23-22} = tsz8_64{3-2}; + let Inst{21} = 0b1; + let Inst{20-19} = tsz8_64{1-0}; + let Inst{18-16} = imm{2-0}; // imm3 + let Inst{15-10} = 0b001101; + let Inst{9-5} = Zm; + let Inst{4-0} = Zdn; + + let Constraints = "$Zdn = $_Zdn"; + let DestructiveInstType = Destructive; + let ElementSize = ElementSizeNone; +} + +multiclass sve2_int_rotate_right_imm<string asm> { + def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; + def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { + let Inst{19} = imm{3}; + } + def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { + let Inst{20-19} = imm{4-3}; + } + def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { + let Inst{22} = imm{5}; + let Inst{20-19} = imm{4-3}; + } +} //===----------------------------------------------------------------------===// // SVE Integer Wide Immediate - Predicated Group @@ -2266,6 +3270,32 @@ multiclass sve_int_while8_rr<bits<3> opc, string asm> { def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; } +class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, + PPRRegOp pprty> +: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), + asm, "\t$Pd, $Rn, $Rm", + "", []>, Sched<[]> { + bits<4> Pd; + bits<5> Rm; + bits<5> Rn; + let Inst{31-24} = 0b00100101; + let Inst{23-22} = sz8_64; + let Inst{21} = 0b1; + let Inst{20-16} = Rm; + let Inst{15-10} = 0b001100; + let Inst{9-5} = Rn; + let Inst{4} = rw; + let Inst{3-0} = Pd; + + let Defs = [NZCV]; +} + +multiclass sve2_int_while_rr<bits<1> rw, string asm> { + def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; + def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; + def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; + def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; +} //===----------------------------------------------------------------------===// // SVE Floating Point Fast Reduction Group @@ -2497,9 +3527,9 @@ multiclass sve_int_index_rr<string asm> { //===----------------------------------------------------------------------===// // SVE Bitwise Shift - Predicated Group //===----------------------------------------------------------------------===// -class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<3> opc, string asm, - ZPRRegOp zprty, Operand immtype, - ElementSizeEnum size> +class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, + ZPRRegOp zprty, Operand immtype, + ElementSizeEnum size> : I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", "", @@ -2509,8 +3539,8 @@ class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<3> opc, string asm, bits<6> imm; let Inst{31-24} = 0b00000100; let Inst{23-22} = tsz8_64{3-2}; - let Inst{21-19} = 0b000; - let Inst{18-16} = opc; + let Inst{21-20} = 0b00; + let Inst{19-16} = opc; let Inst{15-13} = 0b100; let Inst{12-10} = Pg; let Inst{9-8} = tsz8_64{1-0}; @@ -2522,7 +3552,7 @@ class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<3> opc, string asm, let ElementSize = size; } -multiclass sve_int_bin_pred_shift_imm_left<bits<3> opc, string asm> { +multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm> { def _B : sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8, ElementSizeB>; def _H : sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16, @@ -2540,7 +3570,7 @@ multiclass sve_int_bin_pred_shift_imm_left<bits<3> opc, string asm> { } } -multiclass sve_int_bin_pred_shift_imm_right<bits<3> opc, string asm> { +multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm> { def _B : sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8, ElementSizeB>; def _H : sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16, @@ -2856,6 +3886,43 @@ multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; } +class sve2_mem_cstnt_vs_base<bits<3> opc, dag iops, string asm, + RegisterOperand VecList> +: I<(outs VecList:$Zt), iops, + asm, "\t$Zt, $Pg, [$Zn, $Rm]", + "", + []>, Sched<[]> { + bits<3> Pg; + bits<5> Rm; + bits<5> Zn; + bits<5> Zt; + let Inst{31-25} = 0b1110010; + let Inst{24-22} = opc; + let Inst{21} = 0b0; + let Inst{20-16} = Rm; + let Inst{15-13} = 0b001; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4-0} = Zt; + + let mayStore = 1; +} + +multiclass sve2_mem_cstnt_vs<bits<3> opc, string asm, + RegisterOperand listty, ZPRRegOp zprty> { + def _REAL : sve2_mem_cstnt_vs_base<opc, (ins PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), + asm, listty>; + + def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", + (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 0>; + def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", + (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, zprty:$Zn, XZR), 0>; + def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", + (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 0>; + def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", + (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, XZR), 1>; +} + class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, RegisterOperand VecList, RegisterOperand zprext> : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), @@ -3304,6 +4371,30 @@ multiclass sve_int_perm_splice<string asm> { def _D : sve_int_perm_splice<0b11, asm, ZPR64>; } +class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, + ZPRRegOp zprty, RegisterOperand VecList> +: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), + asm, "\t$Zd, $Pg, $Zn", + "", + []>, Sched<[]> { + bits<3> Pg; + bits<5> Zn; + bits<5> Zd; + let Inst{31-24} = 0b00000101; + let Inst{23-22} = sz8_64; + let Inst{21-13} = 0b101101100; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_int_perm_splice_cons<string asm> { + def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; + def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; + def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; + def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; +} + class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty> : I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), @@ -4003,6 +5094,46 @@ multiclass sve_mem_p_fill<string asm> { (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; } +class sve2_mem_cldnt_vs_base<bits<5> opc, dag iops, string asm, + RegisterOperand VecList> +: I<(outs VecList:$Zt), iops, + asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", + "", + []>, Sched<[]> { + bits<3> Pg; + bits<5> Rm; + bits<5> Zn; + bits<5> Zt; + let Inst{31} = 0b1; + let Inst{30} = opc{4}; + let Inst{29-25} = 0b00010; + let Inst{24-23} = opc{3-2}; + let Inst{22-21} = 0b00; + let Inst{20-16} = Rm; + let Inst{15} = 0b1; + let Inst{14-13} = opc{1-0}; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4-0} = Zt; + + let mayLoad = 1; +} + +multiclass sve2_mem_cldnt_vs<bits<5> opc, string asm, + RegisterOperand listty, ZPRRegOp zprty> { + def _REAL : sve2_mem_cldnt_vs_base<opc, (ins PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), + asm, listty>; + + def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", + (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 0>; + def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", + (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, zprty:$Zn, XZR), 0>; + def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", + (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 0>; + def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", + (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, XZR), 1>; +} + //===----------------------------------------------------------------------===// // SVE Memory - 64-bit Gather Group //===----------------------------------------------------------------------===// @@ -4454,3 +5585,132 @@ multiclass sve_int_break_z<bits<3> opc, string asm> { def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; } +//===----------------------------------------------------------------------===// +// SVE2 String Processing Group +//===----------------------------------------------------------------------===// + +class sve2_char_match<bit sz, bit opc, string asm, + PPRRegOp pprty, ZPRRegOp zprty> +: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), + asm, "\t$Pd, $Pg/z, $Zn, $Zm", + "", + []>, Sched<[]> { + bits<4> Pd; + bits<3> Pg; + bits<5> Zm; + bits<5> Zn; + let Inst{31-23} = 0b010001010; + let Inst{22} = sz; + let Inst{21} = 0b1; + let Inst{20-16} = Zm; + let Inst{15-13} = 0b100; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4} = opc; + let Inst{3-0} = Pd; + + let Defs = [NZCV]; +} + +multiclass sve2_char_match<bit opc, string asm> { + def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; + def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Histogram Computation - Segment Group +//===----------------------------------------------------------------------===// + +class sve2_hist_gen_segment<string asm> +: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), + asm, "\t$Zd, $Zn, $Zm", + "", + []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<5> Zm; + let Inst{31-21} = 0b01000101001; + let Inst{20-16} = Zm; + let Inst{15-10} = 0b101000; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +//===----------------------------------------------------------------------===// +// SVE2 Histogram Computation - Vector Group +//===----------------------------------------------------------------------===// + +class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> +: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), + asm, "\t$Zd, $Pg/z, $Zn, $Zm", + "", + []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<3> Pg; + bits<5> Zm; + let Inst{31-23} = 0b010001011; + let Inst{22} = sz; + let Inst{21} = 0b1; + let Inst{20-16} = Zm; + let Inst{15-13} = 0b110; + let Inst{12-10} = Pg; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +multiclass sve2_hist_gen_vector<string asm> { + def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; + def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; +} + +//===----------------------------------------------------------------------===// +// SVE2 Crypto Extensions Group +//===----------------------------------------------------------------------===// + +class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> +: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), + asm, "\t$Zd, $Zn, $Zm", + "", + []>, Sched<[]> { + bits<5> Zd; + bits<5> Zn; + bits<5> Zm; + let Inst{31-21} = 0b01000101001; + let Inst{20-16} = Zm; + let Inst{15-11} = 0b11110; + let Inst{10} = opc; + let Inst{9-5} = Zn; + let Inst{4-0} = Zd; +} + +class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> +: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), + asm, "\t$Zdn, $_Zdn, $Zm", + "", + []>, Sched<[]> { + bits<5> Zdn; + bits<5> Zm; + let Inst{31-17} = 0b010001010010001; + let Inst{16} = opc{1}; + let Inst{15-11} = 0b11100; + let Inst{10} = opc{0}; + let Inst{9-5} = Zm; + let Inst{4-0} = Zdn; + + let Constraints = "$Zdn = $_Zdn"; +} + +class sve2_crypto_unary_op<bit opc, string asm> +: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn), + asm, "\t$Zdn, $_Zdn", + "", + []>, Sched<[]> { + bits<5> Zdn; + let Inst{31-11} = 0b010001010010000011100; + let Inst{10} = opc; + let Inst{9-5} = 0b00000; + let Inst{4-0} = Zdn; + + let Constraints = "$Zdn = $_Zdn"; +} diff --git a/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp index 8fb161574c5b..7f02da6a9516 100644 --- a/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp +++ b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp @@ -1,39 +1,50 @@ //===-- AArch64TargetInfo.cpp - AArch64 Target Implementation -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include "llvm/ADT/Triple.h" +#include "TargetInfo/AArch64TargetInfo.h" #include "llvm/Support/TargetRegistry.h" + using namespace llvm; -namespace llvm { -Target &getTheAArch64leTarget() { +Target &llvm::getTheAArch64leTarget() { static Target TheAArch64leTarget; return TheAArch64leTarget; } -Target &getTheAArch64beTarget() { +Target &llvm::getTheAArch64beTarget() { static Target TheAArch64beTarget; return TheAArch64beTarget; } -Target &getTheARM64Target() { +Target &llvm::getTheAArch64_32Target() { + static Target TheAArch64leTarget; + return TheAArch64leTarget; +} +Target &llvm::getTheARM64Target() { static Target TheARM64Target; return TheARM64Target; } -} // namespace llvm +Target &llvm::getTheARM64_32Target() { + static Target TheARM64_32Target; + return TheARM64_32Target; +} extern "C" void LLVMInitializeAArch64TargetInfo() { // Now register the "arm64" name for use with "-march". We don't want it to - // take possession of the Triple::aarch64 tag though. + // take possession of the Triple::aarch64 tags though. TargetRegistry::RegisterTarget(getTheARM64Target(), "arm64", "ARM64 (little endian)", "AArch64", [](Triple::ArchType) { return false; }, true); + TargetRegistry::RegisterTarget(getTheARM64_32Target(), "arm64_32", + "ARM64 (little endian ILP32)", "AArch64", + [](Triple::ArchType) { return false; }, true); RegisterTarget<Triple::aarch64, /*HasJIT=*/true> Z( getTheAArch64leTarget(), "aarch64", "AArch64 (little endian)", "AArch64"); RegisterTarget<Triple::aarch64_be, /*HasJIT=*/true> W( getTheAArch64beTarget(), "aarch64_be", "AArch64 (big endian)", "AArch64"); + RegisterTarget<Triple::aarch64_32, /*HasJIT=*/true> X( + getTheAArch64_32Target(), "aarch64_32", "AArch64 (little endian ILP32)", "AArch64"); } diff --git a/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.h b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.h new file mode 100644 index 000000000000..b3728a11bb5d --- /dev/null +++ b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.h @@ -0,0 +1,24 @@ +//===-- AArch64TargetInfo.h - AArch64 Target Implementation -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AARCH64_TARGETINFO_AARCH64TARGETINFO_H +#define LLVM_LIB_TARGET_AARCH64_TARGETINFO_AARCH64TARGETINFO_H + +namespace llvm { + +class Target; + +Target &getTheAArch64leTarget(); +Target &getTheAArch64beTarget(); +Target &getTheAArch64_32Target(); +Target &getTheARM64Target(); +Target &getTheARM64_32Target(); + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_AARCH64_TARGETINFO_AARCH64TARGETINFO_H diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp index c88155db7037..7bb075c36e79 100644 --- a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp +++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp @@ -1,9 +1,8 @@ //===-- AArch64BaseInfo.cpp - AArch64 Base encoding information------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/lib/Target/AArch64/Utils/AArch64BaseInfo.h index 44c6a6b44895..e5e2fc2cb0df 100644 --- a/lib/Target/AArch64/Utils/AArch64BaseInfo.h +++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.h @@ -1,9 +1,8 @@ //===-- AArch64BaseInfo.h - Top level definitions for AArch64 ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -186,6 +185,49 @@ static inline unsigned getDRegFromBReg(unsigned Reg) { return Reg; } +static inline bool atomicBarrierDroppedOnZero(unsigned Opcode) { + switch (Opcode) { + case AArch64::LDADDAB: case AArch64::LDADDAH: + case AArch64::LDADDAW: case AArch64::LDADDAX: + case AArch64::LDADDALB: case AArch64::LDADDALH: + case AArch64::LDADDALW: case AArch64::LDADDALX: + case AArch64::LDCLRAB: case AArch64::LDCLRAH: + case AArch64::LDCLRAW: case AArch64::LDCLRAX: + case AArch64::LDCLRALB: case AArch64::LDCLRALH: + case AArch64::LDCLRALW: case AArch64::LDCLRALX: + case AArch64::LDEORAB: case AArch64::LDEORAH: + case AArch64::LDEORAW: case AArch64::LDEORAX: + case AArch64::LDEORALB: case AArch64::LDEORALH: + case AArch64::LDEORALW: case AArch64::LDEORALX: + case AArch64::LDSETAB: case AArch64::LDSETAH: + case AArch64::LDSETAW: case AArch64::LDSETAX: + case AArch64::LDSETALB: case AArch64::LDSETALH: + case AArch64::LDSETALW: case AArch64::LDSETALX: + case AArch64::LDSMAXAB: case AArch64::LDSMAXAH: + case AArch64::LDSMAXAW: case AArch64::LDSMAXAX: + case AArch64::LDSMAXALB: case AArch64::LDSMAXALH: + case AArch64::LDSMAXALW: case AArch64::LDSMAXALX: + case AArch64::LDSMINAB: case AArch64::LDSMINAH: + case AArch64::LDSMINAW: case AArch64::LDSMINAX: + case AArch64::LDSMINALB: case AArch64::LDSMINALH: + case AArch64::LDSMINALW: case AArch64::LDSMINALX: + case AArch64::LDUMAXAB: case AArch64::LDUMAXAH: + case AArch64::LDUMAXAW: case AArch64::LDUMAXAX: + case AArch64::LDUMAXALB: case AArch64::LDUMAXALH: + case AArch64::LDUMAXALW: case AArch64::LDUMAXALX: + case AArch64::LDUMINAB: case AArch64::LDUMINAH: + case AArch64::LDUMINAW: case AArch64::LDUMINAX: + case AArch64::LDUMINALB: case AArch64::LDUMINALH: + case AArch64::LDUMINALW: case AArch64::LDUMINALX: + case AArch64::SWPAB: case AArch64::SWPAH: + case AArch64::SWPAW: case AArch64::SWPAX: + case AArch64::SWPALB: case AArch64::SWPALH: + case AArch64::SWPALW: case AArch64::SWPALX: + return true; + } + return false; +} + namespace AArch64CC { // The CondCodes constants map directly to the 4-bit encoding of the condition |