diff options
Diffstat (limited to 'lib/Target/AArch64/AArch64CallLowering.cpp')
| -rw-r--r-- | lib/Target/AArch64/AArch64CallLowering.cpp | 205 |
1 files changed, 128 insertions, 77 deletions
diff --git a/lib/Target/AArch64/AArch64CallLowering.cpp b/lib/Target/AArch64/AArch64CallLowering.cpp index 5980e5684e89..59757769c89a 100644 --- a/lib/Target/AArch64/AArch64CallLowering.cpp +++ b/lib/Target/AArch64/AArch64CallLowering.cpp @@ -1,9 +1,8 @@ //===--- AArch64CallLowering.cpp - Call lowering --------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -45,6 +44,8 @@ #include <cstdint> #include <iterator> +#define DEBUG_TYPE "aarch64-call-lowering" + using namespace llvm; AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI) @@ -56,18 +57,18 @@ struct IncomingArgHandler : public CallLowering::ValueHandler { CCAssignFn *AssignFn) : ValueHandler(MIRBuilder, MRI, AssignFn), StackUsed(0) {} - unsigned getStackAddress(uint64_t Size, int64_t Offset, + Register getStackAddress(uint64_t Size, int64_t Offset, MachinePointerInfo &MPO) override { auto &MFI = MIRBuilder.getMF().getFrameInfo(); int FI = MFI.CreateFixedObject(Size, Offset, true); MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); - unsigned AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64)); + Register AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64)); MIRBuilder.buildFrameIndex(AddrReg, FI); StackUsed = std::max(StackUsed, Size + Offset); return AddrReg; } - void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign &VA) override { markPhysRegUsed(PhysReg); switch (VA.getLocInfo()) { @@ -84,11 +85,12 @@ struct IncomingArgHandler : public CallLowering::ValueHandler { } } - void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size, + void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) override { + // FIXME: Get alignment auto MMO = MIRBuilder.getMF().getMachineMemOperand( MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, - 0); + 1); MIRBuilder.buildLoad(ValVReg, Addr, *MMO); } @@ -97,6 +99,8 @@ struct IncomingArgHandler : public CallLowering::ValueHandler { /// (it's an implicit-def of the BL). virtual void markPhysRegUsed(unsigned PhysReg) = 0; + bool isArgumentHandler() const override { return true; } + uint64_t StackUsed; }; @@ -129,31 +133,31 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler { : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB), AssignFnVarArg(AssignFnVarArg), StackSize(0) {} - unsigned getStackAddress(uint64_t Size, int64_t Offset, + Register getStackAddress(uint64_t Size, int64_t Offset, MachinePointerInfo &MPO) override { LLT p0 = LLT::pointer(0, 64); LLT s64 = LLT::scalar(64); - unsigned SPReg = MRI.createGenericVirtualRegister(p0); - MIRBuilder.buildCopy(SPReg, AArch64::SP); + Register SPReg = MRI.createGenericVirtualRegister(p0); + MIRBuilder.buildCopy(SPReg, Register(AArch64::SP)); - unsigned OffsetReg = MRI.createGenericVirtualRegister(s64); + Register OffsetReg = MRI.createGenericVirtualRegister(s64); MIRBuilder.buildConstant(OffsetReg, Offset); - unsigned AddrReg = MRI.createGenericVirtualRegister(p0); + Register AddrReg = MRI.createGenericVirtualRegister(p0); MIRBuilder.buildGEP(AddrReg, SPReg, OffsetReg); MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); return AddrReg; } - void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign &VA) override { MIB.addUse(PhysReg, RegState::Implicit); - unsigned ExtReg = extendRegister(ValVReg, VA); + Register ExtReg = extendRegister(ValVReg, VA); MIRBuilder.buildCopy(PhysReg, ExtReg); } - void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size, + void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) override { if (VA.getLocInfo() == CCValAssign::LocInfo::AExt) { Size = VA.getLocVT().getSizeInBits() / 8; @@ -162,7 +166,7 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler { .getReg(); } auto MMO = MIRBuilder.getMF().getMachineMemOperand( - MPO, MachineMemOperand::MOStore, Size, 0); + MPO, MachineMemOperand::MOStore, Size, 1); MIRBuilder.buildStore(ValVReg, Addr, *MMO); } @@ -188,8 +192,7 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler { void AArch64CallLowering::splitToValueTypes( const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs, - const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv, - const SplitArgTy &PerformArgSplit) const { + const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv) const { const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>(); LLVMContext &Ctx = OrigArg.Ty->getContext(); @@ -203,32 +206,31 @@ void AArch64CallLowering::splitToValueTypes( if (SplitVTs.size() == 1) { // No splitting to do, but we want to replace the original type (e.g. [1 x // double] -> double). - SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx), + SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx), OrigArg.Flags, OrigArg.IsFixed); return; } - unsigned FirstRegIdx = SplitArgs.size(); + // Create one ArgInfo for each virtual register in the original ArgInfo. + assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch"); + bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( OrigArg.Ty, CallConv, false); - for (auto SplitVT : SplitVTs) { - Type *SplitTy = SplitVT.getTypeForEVT(Ctx); - SplitArgs.push_back( - ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*SplitTy, DL)), - SplitTy, OrigArg.Flags, OrigArg.IsFixed}); + for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { + Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx); + SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.Flags, + OrigArg.IsFixed); if (NeedsRegBlock) SplitArgs.back().Flags.setInConsecutiveRegs(); } SplitArgs.back().Flags.setInConsecutiveRegsLast(); - - for (unsigned i = 0; i < Offsets.size(); ++i) - PerformArgSplit(SplitArgs[FirstRegIdx + i].Reg, Offsets[i] * 8); } bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, - ArrayRef<unsigned> VRegs) const { + ArrayRef<Register> VRegs, + Register SwiftErrorVReg) const { auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR); assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && "Return value without a vreg"); @@ -250,34 +252,101 @@ bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, "For each split Type there should be exactly one VReg."); SmallVector<ArgInfo, 8> SplitArgs; + CallingConv::ID CC = F.getCallingConv(); + for (unsigned i = 0; i < SplitEVTs.size(); ++i) { - // We zero-extend i1s to i8. - unsigned CurVReg = VRegs[i]; - if (MRI.getType(VRegs[i]).getSizeInBits() == 1) { - CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg) - ->getOperand(0) - .getReg(); + if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) > 1) { + LLVM_DEBUG(dbgs() << "Can't handle extended arg types which need split"); + return false; } + Register CurVReg = VRegs[i]; ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx)}; setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); - splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, F.getCallingConv(), - [&](unsigned Reg, uint64_t Offset) { - MIRBuilder.buildExtract(Reg, CurVReg, Offset); - }); + + // i1 is a special case because SDAG i1 true is naturally zero extended + // when widened using ANYEXT. We need to do it explicitly here. + if (MRI.getType(CurVReg).getSizeInBits() == 1) { + CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0); + } else { + // Some types will need extending as specified by the CC. + MVT NewVT = TLI.getRegisterTypeForCallingConv(Ctx, CC, SplitEVTs[i]); + if (EVT(NewVT) != SplitEVTs[i]) { + unsigned ExtendOp = TargetOpcode::G_ANYEXT; + if (F.getAttributes().hasAttribute(AttributeList::ReturnIndex, + Attribute::SExt)) + ExtendOp = TargetOpcode::G_SEXT; + else if (F.getAttributes().hasAttribute(AttributeList::ReturnIndex, + Attribute::ZExt)) + ExtendOp = TargetOpcode::G_ZEXT; + + LLT NewLLT(NewVT); + LLT OldLLT(MVT::getVT(CurArgInfo.Ty)); + CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Ctx); + // Instead of an extend, we might have a vector type which needs + // padding with more elements, e.g. <2 x half> -> <4 x half>. + if (NewVT.isVector()) { + if (OldLLT.isVector()) { + if (NewLLT.getNumElements() > OldLLT.getNumElements()) { + // We don't handle VA types which are not exactly twice the + // size, but can easily be done in future. + if (NewLLT.getNumElements() != OldLLT.getNumElements() * 2) { + LLVM_DEBUG(dbgs() << "Outgoing vector ret has too many elts"); + return false; + } + auto Undef = MIRBuilder.buildUndef({OldLLT}); + CurVReg = + MIRBuilder.buildMerge({NewLLT}, {CurVReg, Undef.getReg(0)}) + .getReg(0); + } else { + // Just do a vector extend. + CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}) + .getReg(0); + } + } else if (NewLLT.getNumElements() == 2) { + // We need to pad a <1 x S> type to <2 x S>. Since we don't have + // <1 x S> vector types in GISel we use a build_vector instead + // of a vector merge/concat. + auto Undef = MIRBuilder.buildUndef({OldLLT}); + CurVReg = + MIRBuilder + .buildBuildVector({NewLLT}, {CurVReg, Undef.getReg(0)}) + .getReg(0); + } else { + LLVM_DEBUG(dbgs() << "Could not handle ret ty"); + return false; + } + } else { + // A scalar extend. + CurVReg = + MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}).getReg(0); + } + } + } + if (CurVReg != CurArgInfo.Regs[0]) { + CurArgInfo.Regs[0] = CurVReg; + // Reset the arg flags after modifying CurVReg. + setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); + } + splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, CC); } OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFn, AssignFn); Success = handleAssignments(MIRBuilder, SplitArgs, Handler); } + if (SwiftErrorVReg) { + MIB.addUse(AArch64::X21, RegState::Implicit); + MIRBuilder.buildCopy(AArch64::X21, SwiftErrorVReg); + } + MIRBuilder.insertInstr(MIB); return Success; } -bool AArch64CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, - const Function &F, - ArrayRef<unsigned> VRegs) const { +bool AArch64CallLowering::lowerFormalArguments( + MachineIRBuilder &MIRBuilder, const Function &F, + ArrayRef<ArrayRef<Register>> VRegs) const { MachineFunction &MF = MIRBuilder.getMF(); MachineBasicBlock &MBB = MIRBuilder.getMBB(); MachineRegisterInfo &MRI = MF.getRegInfo(); @@ -288,26 +357,11 @@ bool AArch64CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, for (auto &Arg : F.args()) { if (DL.getTypeStoreSize(Arg.getType()) == 0) continue; + ArgInfo OrigArg{VRegs[i], Arg.getType()}; setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F); - bool Split = false; - LLT Ty = MRI.getType(VRegs[i]); - unsigned Dst = VRegs[i]; - - splitToValueTypes(OrigArg, SplitArgs, DL, MRI, F.getCallingConv(), - [&](unsigned Reg, uint64_t Offset) { - if (!Split) { - Split = true; - Dst = MRI.createGenericVirtualRegister(Ty); - MIRBuilder.buildUndef(Dst); - } - unsigned Tmp = MRI.createGenericVirtualRegister(Ty); - MIRBuilder.buildInsert(Tmp, Dst, Reg, Offset); - Dst = Tmp; - }); - - if (Dst != VRegs[i]) - MIRBuilder.buildCopy(VRegs[i], Dst); + + splitToValueTypes(OrigArg, SplitArgs, DL, MRI, F.getCallingConv()); ++i; } @@ -351,7 +405,8 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, const MachineOperand &Callee, const ArgInfo &OrigRet, - ArrayRef<ArgInfo> OrigArgs) const { + ArrayRef<ArgInfo> OrigArgs, + Register SwiftErrorVReg) const { MachineFunction &MF = MIRBuilder.getMF(); const Function &F = MF.getFunction(); MachineRegisterInfo &MRI = MF.getRegInfo(); @@ -359,10 +414,10 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, SmallVector<ArgInfo, 8> SplitArgs; for (auto &OrigArg : OrigArgs) { - splitToValueTypes(OrigArg, SplitArgs, DL, MRI, CallConv, - [&](unsigned Reg, uint64_t Offset) { - MIRBuilder.buildExtract(Reg, OrigArg.Reg, Offset); - }); + splitToValueTypes(OrigArg, SplitArgs, DL, MRI, CallConv); + // AAPCS requires that we zero-extend i1 to 8 bits by the caller. + if (OrigArg.Ty->isIntegerTy(1)) + SplitArgs.back().Flags.setZExt(); } // Find out which ABI gets to decide where things go. @@ -412,23 +467,19 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, // symmetry with the arugments, the physical register must be an // implicit-define of the call instruction. CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(F.getCallingConv()); - if (OrigRet.Reg) { + if (!OrigRet.Ty->isVoidTy()) { SplitArgs.clear(); - SmallVector<uint64_t, 8> RegOffsets; - SmallVector<unsigned, 8> SplitRegs; - splitToValueTypes(OrigRet, SplitArgs, DL, MRI, F.getCallingConv(), - [&](unsigned Reg, uint64_t Offset) { - RegOffsets.push_back(Offset); - SplitRegs.push_back(Reg); - }); + splitToValueTypes(OrigRet, SplitArgs, DL, MRI, F.getCallingConv()); CallReturnHandler Handler(MIRBuilder, MRI, MIB, RetAssignFn); if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) return false; + } - if (!RegOffsets.empty()) - MIRBuilder.buildSequence(OrigRet.Reg, SplitRegs, RegOffsets); + if (SwiftErrorVReg) { + MIB.addDef(AArch64::X21, RegState::Implicit); + MIRBuilder.buildCopy(SwiftErrorVReg, Register(AArch64::X21)); } CallSeqStart.addImm(Handler.StackSize).addImm(0); |