diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2021-12-25 22:30:44 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2021-12-25 22:30:44 +0000 |
| commit | 77fc4c146f0870ffb09c1afb823ccbe742c5e6ff (patch) | |
| tree | 5c0eb39553003b9c75a901af6bc4ddabd6f2f28c /llvm/lib/Target/ARM | |
| parent | f65dcba83ce5035ab88a85fe17628b447eb56e1b (diff) | |
Diffstat (limited to 'llvm/lib/Target/ARM')
27 files changed, 725 insertions, 298 deletions
diff --git a/llvm/lib/Target/ARM/A15SDOptimizer.cpp b/llvm/lib/Target/ARM/A15SDOptimizer.cpp index f4d0f4a6d6b0..d0efecad63bc 100644 --- a/llvm/lib/Target/ARM/A15SDOptimizer.cpp +++ b/llvm/lib/Target/ARM/A15SDOptimizer.cpp @@ -592,16 +592,15 @@ bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) { SmallVector<unsigned, 8> Defs = getReadDPRs(MI); bool Modified = false; - for (SmallVectorImpl<unsigned>::iterator I = Defs.begin(), E = Defs.end(); - I != E; ++I) { + for (unsigned I : Defs) { // Follow the def-use chain for this DPR through COPYs, and also through // PHIs (which are essentially multi-way COPYs). It is because of PHIs that // we can end up with multiple defs of this DPR. SmallVector<MachineInstr *, 8> DefSrcs; - if (!Register::isVirtualRegister(*I)) + if (!Register::isVirtualRegister(I)) continue; - MachineInstr *Def = MRI->getVRegDef(*I); + MachineInstr *Def = MRI->getVRegDef(I); if (!Def) continue; @@ -628,18 +627,17 @@ bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) { if (NewReg != 0) { Modified = true; - for (SmallVectorImpl<MachineOperand *>::const_iterator I = Uses.begin(), - E = Uses.end(); I != E; ++I) { + for (MachineOperand *Use : Uses) { // Make sure to constrain the register class of the new register to // match what we're replacing. Otherwise we can optimize a DPR_VFP2 // reference into a plain DPR, and that will end poorly. NewReg is // always virtual here, so there will always be a matching subclass // to find. - MRI->constrainRegClass(NewReg, MRI->getRegClass((*I)->getReg())); + MRI->constrainRegClass(NewReg, MRI->getRegClass(Use->getReg())); - LLVM_DEBUG(dbgs() << "Replacing operand " << **I << " with " + LLVM_DEBUG(dbgs() << "Replacing operand " << *Use << " with " << printReg(NewReg) << "\n"); - (*I)->substVirtReg(NewReg, 0, *TRI); + Use->substVirtReg(NewReg, 0, *TRI); } } Replacements[MI] = NewReg; diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td index e03dd597eb65..8173fe4036a8 100644 --- a/llvm/lib/Target/ARM/ARM.td +++ b/llvm/lib/Target/ARM/ARM.td @@ -446,6 +446,11 @@ def FeaturePACBTI : SubtargetFeature<"pacbti", "HasPACBTI", "true", "Enable Pointer Authentication and Branch " "Target Identification">; +def FeatureNoBTIAtReturnTwice : SubtargetFeature<"no-bti-at-return-twice", + "NoBTIAtReturnTwice", "true", + "Don't place a BTI instruction " + "after a return-twice">; + //===----------------------------------------------------------------------===// // ARM architecture class // diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp index 6a88ac485e69..fa09b2567aa9 100644 --- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp +++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp @@ -1153,8 +1153,12 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { unsigned StartOp = 2 + 2; // Use all the operands. unsigned NumOffset = 0; - // Amount of SP adjustment folded into a push. - unsigned Pad = 0; + // Amount of SP adjustment folded into a push, before the + // registers are stored (pad at higher addresses). + unsigned PadBefore = 0; + // Amount of SP adjustment folded into a push, after the + // registers are stored (pad at lower addresses). + unsigned PadAfter = 0; switch (Opc) { default: @@ -1185,7 +1189,7 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { "Pad registers must come before restored ones"); unsigned Width = TargetRegInfo->getRegSizeInBits(MO.getReg(), MachineRegInfo) / 8; - Pad += Width; + PadAfter += Width; continue; } // Check for registers that are remapped (for a Thumb1 prologue that @@ -1201,14 +1205,32 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { case ARM::t2STR_PRE: assert(MI->getOperand(2).getReg() == ARM::SP && "Only stack pointer as a source reg is supported"); + if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg)) + SrcReg = RemappedReg; + + RegList.push_back(SrcReg); + break; + case ARM::t2STRD_PRE: + assert(MI->getOperand(3).getReg() == ARM::SP && + "Only stack pointer as a source reg is supported"); + SrcReg = MI->getOperand(1).getReg(); + if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg)) + SrcReg = RemappedReg; + RegList.push_back(SrcReg); + SrcReg = MI->getOperand(2).getReg(); + if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg)) + SrcReg = RemappedReg; RegList.push_back(SrcReg); + PadBefore = -MI->getOperand(4).getImm() - 8; break; } if (MAI->getExceptionHandlingType() == ExceptionHandling::ARM) { + if (PadBefore) + ATS.emitPad(PadBefore); ATS.emitRegSave(RegList, Opc == ARM::VSTMDDB_UPD); // Account for the SP adjustment, folded into the push. - if (Pad) - ATS.emitPad(Pad); + if (PadAfter) + ATS.emitPad(PadAfter); } } else { // Changes of stack / frame pointer. @@ -1300,6 +1322,10 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { Offset = MI->getOperand(2).getImm(); AFI->EHPrologueOffsetInRegs[DstReg] |= (Offset << 16); break; + case ARM::t2PAC: + case ARM::t2PACBTI: + AFI->EHPrologueRemappedRegs[ARM::R12] = ARM::RA_AUTH_CODE; + break; default: MI->print(errs()); llvm_unreachable("Unsupported opcode for unwinding information"); diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp index 2a12947d24a8..884f38ff6c58 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -2629,8 +2629,8 @@ bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget, // Add the complete list back in. MachineInstrBuilder MIB(MF, &*MI); - for (int i = RegList.size() - 1; i >= 0; --i) - MIB.add(RegList[i]); + for (const MachineOperand &MO : llvm::reverse(RegList)) + MIB.add(MO); return true; } @@ -5678,7 +5678,7 @@ bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, /// | | Thumb2 | ARM | /// +-------------------------+--------+-----+ /// | Call overhead in Bytes | 4 | 4 | -/// | Frame overhead in Bytes | 4 | 4 | +/// | Frame overhead in Bytes | 2 | 4 | /// | Stack fixup required | No | No | /// +-------------------------+--------+-----+ /// @@ -5755,7 +5755,7 @@ struct OutlinerCosts { CallThunk(target.isThumb() ? 4 : 4), FrameThunk(target.isThumb() ? 0 : 0), CallNoLRSave(target.isThumb() ? 4 : 4), - FrameNoLRSave(target.isThumb() ? 4 : 4), + FrameNoLRSave(target.isThumb() ? 2 : 4), CallRegSave(target.isThumb() ? 8 : 12), FrameRegSave(target.isThumb() ? 2 : 4), CallDefault(target.isThumb() ? 8 : 12), @@ -5868,11 +5868,17 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( return outliner::OutlinedFunction(); } + // We expect the majority of the outlining candidates to be in consensus with + // regard to return address sign and authentication, and branch target + // enforcement, in other words, partitioning according to all the four + // possible combinations of PAC-RET and BTI is going to yield one big subset + // and three small (likely empty) subsets. That allows us to cull incompatible + // candidates separately for PAC-RET and BTI. + // Partition the candidates in two sets: one with BTI enabled and one with BTI - // disabled. Remove the candidates from the smaller set. We expect the - // majority of the candidates to be in consensus with regard to branch target - // enforcement with just a few oddballs, but if they are the same number - // prefer the non-BTI ones for outlining, since they have less overhead. + // disabled. Remove the candidates from the smaller set. If they are the same + // number prefer the non-BTI ones for outlining, since they have less + // overhead. auto NoBTI = llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) { const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>(); @@ -5883,6 +5889,24 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( RepeatedSequenceLocs.erase(NoBTI, RepeatedSequenceLocs.end()); else RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoBTI); + + if (RepeatedSequenceLocs.size() < 2) + return outliner::OutlinedFunction(); + + // Likewise, partition the candidates according to PAC-RET enablement. + auto NoPAC = + llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) { + const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>(); + // If the function happens to not spill the LR, do not disqualify it + // from the outlining. + return AFI.shouldSignReturnAddress(true); + }); + if (std::distance(RepeatedSequenceLocs.begin(), NoPAC) > + std::distance(NoPAC, RepeatedSequenceLocs.end())) + RepeatedSequenceLocs.erase(NoPAC, RepeatedSequenceLocs.end()); + else + RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoPAC); + if (RepeatedSequenceLocs.size() < 2) return outliner::OutlinedFunction(); @@ -5899,6 +5923,7 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( }; OutlinerCosts Costs(Subtarget); + const auto &SomeMFI = *RepeatedSequenceLocs.front().getMF()->getInfo<ARMFunctionInfo>(); // Adjust costs to account for the BTI instructions. @@ -5909,6 +5934,13 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( Costs.FrameTailCall += 4; Costs.FrameThunk += 4; } + + // Adjust costs to account for sign and authentication instructions. + if (SomeMFI.shouldSignReturnAddress(true)) { + Costs.CallDefault += 8; // +PAC instr, +AUT instr + Costs.SaveRestoreLROnStack += 8; // +PAC instr, +AUT instr + } + unsigned FrameID = MachineOutlinerDefault; unsigned NumBytesToCreateFrame = Costs.FrameDefault; @@ -6325,6 +6357,11 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, // * LR is available in the range (No save/restore around call) // * The range doesn't include calls (No save/restore in outlined frame) // are true. + // These conditions also ensure correctness of the return address + // authentication - we insert sign and authentication instructions only if + // we save/restore LR on stack, but then this condition ensures that the + // outlined range does not modify the SP, therefore the SP value used for + // signing is the same as the one used for authentication. // FIXME: This is very restrictive; the flags check the whole block, // not just the bit we will try to outline. bool MightNeedStackFixUp = @@ -6369,23 +6406,39 @@ void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const { } void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const { - unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM; - int Align = -Subtarget.getStackAlignment().value(); - BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP) - .addReg(ARM::LR, RegState::Kill) - .addReg(ARM::SP) - .addImm(Align) - .add(predOps(ARMCC::AL)); -} + MachineBasicBlock::iterator It, bool CFI, + bool Auth) const { + int Align = std::max(Subtarget.getStackAlignment().value(), uint64_t(8)); + assert(Align >= 8 && Align <= 256); + if (Auth) { + assert(Subtarget.isThumb2()); + // Compute PAC in R12. Outlining ensures R12 is dead across the outlined + // sequence. + BuildMI(MBB, It, DebugLoc(), get(ARM::t2PAC)) + .setMIFlags(MachineInstr::FrameSetup); + BuildMI(MBB, It, DebugLoc(), get(ARM::t2STRD_PRE), ARM::SP) + .addReg(ARM::R12, RegState::Kill) + .addReg(ARM::LR, RegState::Kill) + .addReg(ARM::SP) + .addImm(-Align) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameSetup); + } else { + unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM; + BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP) + .addReg(ARM::LR, RegState::Kill) + .addReg(ARM::SP) + .addImm(-Align) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameSetup); + } + + if (!CFI) + return; -void ARMBaseInstrInfo::emitCFIForLRSaveOnStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - int Align = Subtarget.getStackAlignment().value(); - // Add a CFI saying the stack was moved down. + + // Add a CFI, saying CFA is offset by Align bytes from SP. int64_t StackPosEntry = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align)); BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) @@ -6394,11 +6447,23 @@ void ARMBaseInstrInfo::emitCFIForLRSaveOnStack( // Add a CFI saying that the LR that we want to find is now higher than // before. - int64_t LRPosEntry = - MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfLR, -Align)); + int LROffset = Auth ? Align - 4 : Align; + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + int64_t LRPosEntry = MF.addFrameInst( + MCCFIInstruction::createOffset(nullptr, DwarfLR, -LROffset)); BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) .addCFIIndex(LRPosEntry) .setMIFlags(MachineInstr::FrameSetup); + if (Auth) { + // Add a CFI for the location of the return adddress PAC. + unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true); + int64_t RACPosEntry = MF.addFrameInst( + MCCFIInstruction::createOffset(nullptr, DwarfRAC, -Align)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(RACPosEntry) + .setMIFlags(MachineInstr::FrameSetup); + } } void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB, @@ -6416,35 +6481,64 @@ void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB, .setMIFlags(MachineInstr::FrameSetup); } -void ARMBaseInstrInfo::restoreLRFromStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; - MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR) - .addReg(ARM::SP, RegState::Define) - .addReg(ARM::SP); - if (!Subtarget.isThumb()) - MIB.addReg(0); - MIB.addImm(Subtarget.getStackAlignment().value()).add(predOps(ARMCC::AL)); -} +void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB, + MachineBasicBlock::iterator It, + bool CFI, bool Auth) const { + int Align = Subtarget.getStackAlignment().value(); + if (Auth) { + assert(Subtarget.isThumb2()); + // Restore return address PAC and LR. + BuildMI(MBB, It, DebugLoc(), get(ARM::t2LDRD_POST)) + .addReg(ARM::R12, RegState::Define) + .addReg(ARM::LR, RegState::Define) + .addReg(ARM::SP, RegState::Define) + .addReg(ARM::SP) + .addImm(Align) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameDestroy); + // LR authentication is after the CFI instructions, below. + } else { + unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; + MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR) + .addReg(ARM::SP, RegState::Define) + .addReg(ARM::SP); + if (!Subtarget.isThumb()) + MIB.addReg(0); + MIB.addImm(Subtarget.getStackAlignment().value()) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameDestroy); + } -void ARMBaseInstrInfo::emitCFIForLRRestoreFromStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - // Now stack has moved back up... - MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - int64_t StackPosEntry = - MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(StackPosEntry) - .setMIFlags(MachineInstr::FrameDestroy); + if (CFI) { + // Now stack has moved back up... + MachineFunction &MF = *MBB.getParent(); + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + int64_t StackPosEntry = + MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(StackPosEntry) + .setMIFlags(MachineInstr::FrameDestroy); - // ... and we have restored LR. - int64_t LRPosEntry = - MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(LRPosEntry) - .setMIFlags(MachineInstr::FrameDestroy); + // ... and we have restored LR. + int64_t LRPosEntry = + MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(LRPosEntry) + .setMIFlags(MachineInstr::FrameDestroy); + + if (Auth) { + unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true); + int64_t Entry = + MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, DwarfRAC)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(Entry) + .setMIFlags(MachineInstr::FrameDestroy); + } + } + + if (Auth) + BuildMI(MBB, It, DebugLoc(), get(ARM::t2AUT)); } void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg( @@ -6500,8 +6594,11 @@ void ARMBaseInstrInfo::buildOutlinedFrame( MBB.addLiveIn(ARM::LR); // Insert a save before the outlined region - saveLROnStack(MBB, It); - emitCFIForLRSaveOnStack(MBB, It); + bool Auth = OF.Candidates.front() + .getMF() + ->getInfo<ARMFunctionInfo>() + ->shouldSignReturnAddress(true); + saveLROnStack(MBB, It, true, Auth); // Fix up the instructions in the range, since we're going to modify the // stack. @@ -6510,8 +6607,7 @@ void ARMBaseInstrInfo::buildOutlinedFrame( fixupPostOutline(MBB); // Insert a restore before the terminator for the function. Restore LR. - restoreLRFromStack(MBB, Et); - emitCFIForLRRestoreFromStack(MBB, Et); + restoreLRFromStack(MBB, Et, true, Auth); } // If this is a tail call outlined function, then there's already a return. @@ -6590,13 +6686,10 @@ MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall( // We have the default case. Save and restore from SP. if (!MBB.isLiveIn(ARM::LR)) MBB.addLiveIn(ARM::LR); - saveLROnStack(MBB, It); - if (!AFI.isLRSpilled()) - emitCFIForLRSaveOnStack(MBB, It); + bool Auth = !AFI.isLRSpilled() && AFI.shouldSignReturnAddress(true); + saveLROnStack(MBB, It, !AFI.isLRSpilled(), Auth); CallPt = MBB.insert(It, CallMIB); - restoreLRFromStack(MBB, It); - if (!AFI.isLRSpilled()) - emitCFIForLRRestoreFromStack(MBB, It); + restoreLRFromStack(MBB, It, !AFI.isLRSpilled(), Auth); It--; return CallPt; } diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h index 5fa912ae35d7..defce07dd862 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h @@ -377,20 +377,20 @@ private: /// constructing an outlined call if one exists. Returns 0 otherwise. unsigned findRegisterToSaveLRTo(const outliner::Candidate &C) const; - // Adds an instruction which saves the link register on top of the stack into - /// the MachineBasicBlock \p MBB at position \p It. - void saveLROnStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; + /// Adds an instruction which saves the link register on top of the stack into + /// the MachineBasicBlock \p MBB at position \p It. If \p Auth is true, + /// compute and store an authentication code alongiside the link register. + /// If \p CFI is true, emit CFI instructions. + void saveLROnStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator It, + bool CFI, bool Auth) const; /// Adds an instruction which restores the link register from the top the - /// stack into the MachineBasicBlock \p MBB at position \p It. + /// stack into the MachineBasicBlock \p MBB at position \p It. If \p Auth is + /// true, restore an authentication code and authenticate LR. + /// If \p CFI is true, emit CFI instructions. void restoreLRFromStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; - - /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, - /// for the case when the LR is saved on the stack. - void emitCFIForLRSaveOnStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; + MachineBasicBlock::iterator It, bool CFI, + bool Auth) const; /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, /// for the case when the LR is saved in the register \p Reg. @@ -399,11 +399,6 @@ private: Register Reg) const; /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, - /// after the LR is was restored from the stack. - void emitCFIForLRRestoreFromStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; - - /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, /// after the LR is was restored from a register. void emitCFIForLRRestoreFromReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const; diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp index b53efe58e8de..c543d02ff75a 100644 --- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp @@ -530,6 +530,8 @@ getFrameIndexInstrOffset(const MachineInstr *MI, int Idx) const { unsigned ImmIdx = 0; switch (AddrMode) { case ARMII::AddrModeT2_i8: + case ARMII::AddrModeT2_i8neg: + case ARMII::AddrModeT2_i8pos: case ARMII::AddrModeT2_i12: case ARMII::AddrMode_i12: InstrOffs = MI->getOperand(Idx+1).getImm(); @@ -728,6 +730,8 @@ bool ARMBaseRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, bool isSigned = true; switch (AddrMode) { case ARMII::AddrModeT2_i8: + case ARMII::AddrModeT2_i8pos: + case ARMII::AddrModeT2_i8neg: case ARMII::AddrModeT2_i12: // i8 supports only negative, and i12 supports only positive, so // based on Offset sign, consider the appropriate instruction diff --git a/llvm/lib/Target/ARM/ARMBranchTargets.cpp b/llvm/lib/Target/ARM/ARMBranchTargets.cpp index 1091c1f970fa..8ba3e627c039 100644 --- a/llvm/lib/Target/ARM/ARMBranchTargets.cpp +++ b/llvm/lib/Target/ARM/ARMBranchTargets.cpp @@ -108,6 +108,7 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB, bool IsFirstBB) { // Which instruction to insert: BTI or PACBTI unsigned OpCode = ARM::t2BTI; + unsigned MIFlags = 0; // Skip meta instructions, including EH labels auto MBBI = llvm::find_if_not(MBB.instrs(), [](const MachineInstr &MI) { @@ -121,6 +122,7 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB, LLVM_DEBUG(dbgs() << "Removing a 'PAC' instr from BB '" << MBB.getName() << "' to replace with PACBTI\n"); OpCode = ARM::t2PACBTI; + MIFlags = MachineInstr::FrameSetup; auto NextMBBI = std::next(MBBI); MBBI->eraseFromParent(); MBBI = NextMBBI; @@ -131,5 +133,6 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB, << (OpCode == ARM::t2BTI ? "BTI" : "PACBTI") << "' instr into BB '" << MBB.getName() << "'\n"); // Finally, insert a new instruction (either PAC or PACBTI) - BuildMI(MBB, MBBI, MBB.findDebugLoc(MBBI), TII.get(OpCode)); + BuildMI(MBB, MBBI, MBB.findDebugLoc(MBBI), TII.get(OpCode)) + .setMIFlags(MIFlags); } diff --git a/llvm/lib/Target/ARM/ARMCallingConv.cpp b/llvm/lib/Target/ARM/ARMCallingConv.cpp index d8d9ca3b912f..32f3a4a632f5 100644 --- a/llvm/lib/Target/ARM/ARMCallingConv.cpp +++ b/llvm/lib/Target/ARM/ARMCallingConv.cpp @@ -230,10 +230,9 @@ static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT, unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size()); if (RegResult) { - for (SmallVectorImpl<CCValAssign>::iterator It = PendingMembers.begin(); - It != PendingMembers.end(); ++It) { - It->convertToReg(RegResult); - State.addLoc(*It); + for (CCValAssign &PendingMember : PendingMembers) { + PendingMember.convertToReg(RegResult); + State.addLoc(PendingMember); ++RegResult; } PendingMembers.clear(); diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp index c2ca4708c208..a2a4f1f3bdfd 100644 --- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp +++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp @@ -310,8 +310,7 @@ void ARMConstantIslands::verify() { BBInfo[RHS.getNumber()].postOffset(); })); LLVM_DEBUG(dbgs() << "Verifying " << CPUsers.size() << " CP users.\n"); - for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) { - CPUser &U = CPUsers[i]; + for (CPUser &U : CPUsers) { unsigned UserOffset = getUserOffset(U); // Verify offset using the real max displacement without the safety // adjustment. @@ -697,10 +696,9 @@ ARMConstantIslands::findConstPoolEntry(unsigned CPI, std::vector<CPEntry> &CPEs = CPEntries[CPI]; // Number of entries per constpool index should be small, just do a // linear search. - for (unsigned i = 0, e = CPEs.size(); i != e; ++i) { - if (CPEs[i].CPEMI == CPEMI) - return &CPEs[i]; - } + for (CPEntry &CPE : CPEs) + if (CPE.CPEMI == CPEMI) + return &CPE; return nullptr; } @@ -1234,27 +1232,27 @@ int ARMConstantIslands::findInRangeCPEntry(CPUser& U, unsigned UserOffset) { // No. Look for previously created clones of the CPE that are in range. unsigned CPI = getCombinedIndex(CPEMI); std::vector<CPEntry> &CPEs = CPEntries[CPI]; - for (unsigned i = 0, e = CPEs.size(); i != e; ++i) { + for (CPEntry &CPE : CPEs) { // We already tried this one - if (CPEs[i].CPEMI == CPEMI) + if (CPE.CPEMI == CPEMI) continue; // Removing CPEs can leave empty entries, skip - if (CPEs[i].CPEMI == nullptr) + if (CPE.CPEMI == nullptr) continue; - if (isCPEntryInRange(UserMI, UserOffset, CPEs[i].CPEMI, U.getMaxDisp(), - U.NegOk)) { - LLVM_DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#" - << CPEs[i].CPI << "\n"); + if (isCPEntryInRange(UserMI, UserOffset, CPE.CPEMI, U.getMaxDisp(), + U.NegOk)) { + LLVM_DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#" << CPE.CPI + << "\n"); // Point the CPUser node to the replacement - U.CPEMI = CPEs[i].CPEMI; + U.CPEMI = CPE.CPEMI; // Change the CPI in the instruction operand to refer to the clone. for (MachineOperand &MO : UserMI->operands()) if (MO.isCPI()) { - MO.setIndex(CPEs[i].CPI); + MO.setIndex(CPE.CPI); break; } // Adjust the refcount of the clone... - CPEs[i].RefCount++; + CPE.RefCount++; // ...and the original. If we didn't remove the old entry, none of the // addresses changed, so we don't need another pass. return decrementCPEReferenceCount(CPI, CPEMI) ? 2 : 1; @@ -1675,15 +1673,14 @@ void ARMConstantIslands::removeDeadCPEMI(MachineInstr *CPEMI) { /// are zero. bool ARMConstantIslands::removeUnusedCPEntries() { unsigned MadeChange = false; - for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) { - std::vector<CPEntry> &CPEs = CPEntries[i]; - for (unsigned j = 0, ee = CPEs.size(); j != ee; ++j) { - if (CPEs[j].RefCount == 0 && CPEs[j].CPEMI) { - removeDeadCPEMI(CPEs[j].CPEMI); - CPEs[j].CPEMI = nullptr; - MadeChange = true; - } + for (std::vector<CPEntry> &CPEs : CPEntries) { + for (CPEntry &CPE : CPEs) { + if (CPE.RefCount == 0 && CPE.CPEMI) { + removeDeadCPEMI(CPE.CPEMI); + CPE.CPEMI = nullptr; + MadeChange = true; } + } } return MadeChange; } @@ -1829,8 +1826,7 @@ bool ARMConstantIslands::optimizeThumb2Instructions() { bool MadeChange = false; // Shrink ADR and LDR from constantpool. - for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) { - CPUser &U = CPUsers[i]; + for (CPUser &U : CPUsers) { unsigned Opcode = U.MI->getOpcode(); unsigned NewOpc = 0; unsigned Scale = 1; diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp index 7a35f252b22a..fa244786a80d 100644 --- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp +++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp @@ -2160,6 +2160,11 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB, return true; } case ARM::tBXNS_RET: { + // For v8.0-M.Main we need to authenticate LR before clearing FPRs, which + // uses R12 as a scratch register. + if (!STI->hasV8_1MMainlineOps() && AFI->shouldSignReturnAddress()) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(ARM::t2AUT)); + MachineBasicBlock &AfterBB = CMSEClearFPRegs(MBB, MBBI); if (STI->hasV8_1MMainlineOps()) { @@ -2169,6 +2174,9 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB, .addReg(ARM::SP) .addImm(4) .add(predOps(ARMCC::AL)); + + if (AFI->shouldSignReturnAddress()) + BuildMI(AfterBB, AfterBB.end(), DebugLoc(), TII->get(ARM::t2AUT)); } // Clear all GPR that are not a use of the return instruction. @@ -3073,6 +3081,22 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB, MI.eraseFromParent(); return true; } + case ARM::t2CALL_BTI: { + MachineFunction &MF = *MI.getMF(); + MachineInstrBuilder MIB = + BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::tBL)); + MIB.cloneMemRefs(MI); + for (unsigned i = 0; i < MI.getNumOperands(); ++i) + MIB.add(MI.getOperand(i)); + if (MI.isCandidateForCallSiteEntry()) + MF.moveCallSiteInfo(&MI, MIB.getInstr()); + MIBundleBuilder Bundler(MBB, MI); + Bundler.append(MIB); + Bundler.append(BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::t2BTI))); + finalizeBundle(MBB, Bundler.begin(), Bundler.end()); + MI.eraseFromParent(); + return true; + } case ARM::LOADDUAL: case ARM::STOREDUAL: { Register PairReg = MI.getOperand(0).getReg(); diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp index b866cf952ff1..4b59f9cb94ce 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp +++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp @@ -503,20 +503,12 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, StackAdjustingInsts DefCFAOffsetCandidates; bool HasFP = hasFP(MF); - // Allocate the vararg register save area. - if (ArgRegsSaveSize) { - emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize, - MachineInstr::FrameSetup); - DefCFAOffsetCandidates.addInst(std::prev(MBBI), ArgRegsSaveSize, true); - } - if (!AFI->hasStackFrame() && (!STI.isTargetWindows() || !WindowsRequiresStackProbe(MF, NumBytes))) { - if (NumBytes - ArgRegsSaveSize != 0) { - emitSPUpdate(isARM, MBB, MBBI, dl, TII, -(NumBytes - ArgRegsSaveSize), + if (NumBytes != 0) { + emitSPUpdate(isARM, MBB, MBBI, dl, TII, -NumBytes, MachineInstr::FrameSetup); - DefCFAOffsetCandidates.addInst(std::prev(MBBI), - NumBytes - ArgRegsSaveSize, true); + DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes, true); } DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP); return; @@ -562,13 +554,26 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, } } - // Move past FPCXT area. MachineBasicBlock::iterator LastPush = MBB.end(), GPRCS1Push, GPRCS2Push; + + // Move past the PAC computation. + if (AFI->shouldSignReturnAddress()) + LastPush = MBBI++; + + // Move past FPCXT area. if (FPCXTSaveSize > 0) { LastPush = MBBI++; DefCFAOffsetCandidates.addInst(LastPush, FPCXTSaveSize, true); } + // Allocate the vararg register save area. + if (ArgRegsSaveSize) { + emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize, + MachineInstr::FrameSetup); + LastPush = std::prev(MBBI); + DefCFAOffsetCandidates.addInst(LastPush, ArgRegsSaveSize, true); + } + // Move past area 1. if (GPRCS1Size > 0) { GPRCS1Push = LastPush = MBBI++; @@ -788,7 +793,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, case ARM::R11: case ARM::R12: if (STI.splitFramePushPop(MF)) { - unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); + unsigned DwarfReg = MRI->getDwarfRegNum( + Reg == ARM::R12 ? (unsigned)ARM::RA_AUTH_CODE : Reg, true); unsigned Offset = MFI.getObjectOffset(FI); unsigned CFIIndex = MF.addFrameInst( MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset)); @@ -923,8 +929,9 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF, DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); if (!AFI->hasStackFrame()) { - if (NumBytes - ReservedArgStack != 0) - emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ReservedArgStack, + if (NumBytes + IncomingArgStackToRestore != 0) + emitSPUpdate(isARM, MBB, MBBI, dl, TII, + NumBytes + IncomingArgStackToRestore, MachineInstr::FrameDestroy); } else { // Unwind MBBI to point to first LDR / VLDRD. @@ -1007,15 +1014,21 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF, if (AFI->getGPRCalleeSavedArea2Size()) MBBI++; if (AFI->getGPRCalleeSavedArea1Size()) MBBI++; - if (AFI->getFPCXTSaveAreaSize()) MBBI++; - } - if (ReservedArgStack || IncomingArgStackToRestore) { - assert((int)ReservedArgStack + IncomingArgStackToRestore >= 0 && - "attempting to restore negative stack amount"); - emitSPUpdate(isARM, MBB, MBBI, dl, TII, - ReservedArgStack + IncomingArgStackToRestore, - MachineInstr::FrameDestroy); + if (ReservedArgStack || IncomingArgStackToRestore) { + assert((int)ReservedArgStack + IncomingArgStackToRestore >= 0 && + "attempting to restore negative stack amount"); + emitSPUpdate(isARM, MBB, MBBI, dl, TII, + ReservedArgStack + IncomingArgStackToRestore, + MachineInstr::FrameDestroy); + } + + // Validate PAC, It should have been already popped into R12. For CMSE entry + // function, the validation instruction is emitted during expansion of the + // tBXNS_RET, since the validation must use the value of SP at function + // entry, before saving, resp. after restoring, FPCXTNS. + if (AFI->shouldSignReturnAddress() && !AFI->isCmseNSEntryFunction()) + BuildMI(MBB, MBBI, DebugLoc(), STI.getInstrInfo()->get(ARM::t2AUT)); } } @@ -1199,6 +1212,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB, const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); + bool hasPAC = AFI->shouldSignReturnAddress(); DebugLoc DL; bool isTailCall = false; bool isInterrupt = false; @@ -1231,7 +1245,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB, continue; if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt && !isCmseEntry && !isTrap && AFI->getArgumentStackToRestore() == 0 && - STI.hasV5TOps() && MBB.succ_empty()) { + STI.hasV5TOps() && MBB.succ_empty() && !hasPAC) { Reg = ARM::PC; // Fold the return instruction into the LDM. DeleteRet = true; @@ -1580,6 +1594,11 @@ bool ARMFrameLowering::spillCalleeSavedRegisters( ARM::t2STR_PRE : ARM::STR_PRE_IMM; unsigned FltOpc = ARM::VSTMDDB_UPD; unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs(); + // Compute PAC in R12. + if (AFI->shouldSignReturnAddress()) { + BuildMI(MBB, MI, DebugLoc(), STI.getInstrInfo()->get(ARM::t2PAC)) + .setMIFlags(MachineInstr::FrameSetup); + } // Save the non-secure floating point context. if (llvm::any_of(CSI, [](const CalleeSavedInfo &C) { return C.getReg() == ARM::FPCXTNS; @@ -1789,6 +1808,13 @@ bool ARMFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { MF.getInfo<ARMFunctionInfo>()->isCmseNSEntryFunction()) return false; + // We are disabling shrinkwrapping for now when PAC is enabled, as + // shrinkwrapping can cause clobbering of r12 when the PAC code is + // generated. A follow-up patch will fix this in a more performant manner. + if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress( + false /*SpillsLR */)) + return false; + return true; } @@ -2315,6 +2341,26 @@ bool ARMFrameLowering::assignCalleeSavedSpillSlots( CSI.back().setRestored(false); } + // For functions, which sign their return address, upon function entry, the + // return address PAC is computed in R12. Treat R12 as a callee-saved register + // in this case. + const auto &AFI = *MF.getInfo<ARMFunctionInfo>(); + if (AFI.shouldSignReturnAddress()) { + // The order of register must match the order we push them, because the + // PEI assigns frame indices in that order. When compiling for return + // address sign and authenication, we use split push, therefore the orders + // we want are: + // LR, R7, R6, R5, R4, <R12>, R11, R10, R9, R8, D15-D8 + CSI.insert(find_if(CSI, + [=](const auto &CS) { + unsigned Reg = CS.getReg(); + return Reg == ARM::R10 || Reg == ARM::R11 || + Reg == ARM::R8 || Reg == ARM::R9 || + ARM::DPRRegClass.contains(Reg); + }), + CalleeSavedInfo(ARM::R12)); + } + return false; } diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 33d115945614..3d45db349644 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -391,6 +391,7 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); + setOperationAction(ISD::VSELECT, VT, Legal); } setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); @@ -428,7 +429,7 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { } // Predicate types - const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1}; + const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1}; for (auto VT : pTypes) { addRegisterClass(VT, &ARM::VCCRRegClass); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); @@ -445,6 +446,16 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::VSELECT, VT, Expand); setOperationAction(ISD::SELECT, VT, Expand); } + setOperationAction(ISD::SETCC, MVT::v2i1, Expand); + setOperationAction(ISD::TRUNCATE, MVT::v2i1, Expand); + setOperationAction(ISD::AND, MVT::v2i1, Expand); + setOperationAction(ISD::OR, MVT::v2i1, Expand); + setOperationAction(ISD::XOR, MVT::v2i1, Expand); + setOperationAction(ISD::SINT_TO_FP, MVT::v2i1, Expand); + setOperationAction(ISD::UINT_TO_FP, MVT::v2i1, Expand); + setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Expand); + setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Expand); + setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); @@ -1647,6 +1658,7 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const { MAKE_CASE(ARMISD::CALL_PRED) MAKE_CASE(ARMISD::CALL_NOLINK) MAKE_CASE(ARMISD::tSECALL) + MAKE_CASE(ARMISD::t2CALL_BTI) MAKE_CASE(ARMISD::BRCOND) MAKE_CASE(ARMISD::BR_JT) MAKE_CASE(ARMISD::BR2_JT) @@ -1853,8 +1865,10 @@ EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, // MVE has a predicate register. if ((Subtarget->hasMVEIntegerOps() && - (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8)) || - (Subtarget->hasMVEFloatOps() && (VT == MVT::v4f32 || VT == MVT::v8f16))) + (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || + VT == MVT::v16i8)) || + (Subtarget->hasMVEFloatOps() && + (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16))) return MVT::getVectorVT(MVT::i1, VT.getVectorElementCount()); return VT.changeVectorElementTypeToInteger(); } @@ -2308,6 +2322,12 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, bool isCmseNSCall = false; bool isSibCall = false; bool PreferIndirect = false; + bool GuardWithBTI = false; + + // Lower 'returns_twice' calls to a pseudo-instruction. + if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) && + !Subtarget->getNoBTIAtReturnTwice()) + GuardWithBTI = AFI->branchTargetEnforcement(); // Determine whether this is a non-secure function call. if (CLI.CB && CLI.CB->getAttributes().hasFnAttr("cmse_nonsecure_call")) @@ -2713,7 +2733,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // FIXME: handle tail calls differently. unsigned CallOpc; if (Subtarget->isThumb()) { - if (isCmseNSCall) + if (GuardWithBTI) + CallOpc = ARMISD::t2CALL_BTI; + else if (isCmseNSCall) CallOpc = ARMISD::tSECALL; else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps()) CallOpc = ARMISD::CALL_NOLINK; @@ -2930,9 +2952,17 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization( // Indirect tail calls cannot be optimized for Thumb1 if the args // to the call take up r0-r3. The reason is that there are no legal registers // left to hold the pointer to the function to be called. - if (Subtarget->isThumb1Only() && Outs.size() >= 4 && - (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect)) - return false; + // Similarly, if the function uses return address sign and authentication, + // r12 is needed to hold the PAC and is not available to hold the callee + // address. + if (Outs.size() >= 4 && + (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect)) { + if (Subtarget->isThumb1Only()) + return false; + // Conservatively assume the function spills LR. + if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(true)) + return false; + } // Look for obvious safe cases to perform tail call optimization that do not // require ABI changes. This is what gcc calls sibcall. @@ -7616,7 +7646,10 @@ static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, unsigned NumElts = VT.getVectorNumElements(); unsigned BoolMask; unsigned BitsPerBool; - if (NumElts == 4) { + if (NumElts == 2) { + BitsPerBool = 8; + BoolMask = 0xff; + } else if (NumElts == 4) { BitsPerBool = 4; BoolMask = 0xf; } else if (NumElts == 8) { @@ -7699,6 +7732,46 @@ static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, DAG.getConstant(N, DL, MVT::i32)); } +// Returns true if the operation N can be treated as qr instruction variant at +// operand Op. +static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) { + switch (N->getOpcode()) { + case ISD::ADD: + case ISD::MUL: + case ISD::SADDSAT: + case ISD::UADDSAT: + return true; + case ISD::SUB: + case ISD::SSUBSAT: + case ISD::USUBSAT: + return N->getOperand(1).getNode() == Op; + case ISD::INTRINSIC_WO_CHAIN: + switch (N->getConstantOperandVal(0)) { + case Intrinsic::arm_mve_add_predicated: + case Intrinsic::arm_mve_mul_predicated: + case Intrinsic::arm_mve_qadd_predicated: + case Intrinsic::arm_mve_vhadd: + case Intrinsic::arm_mve_hadd_predicated: + case Intrinsic::arm_mve_vqdmulh: + case Intrinsic::arm_mve_qdmulh_predicated: + case Intrinsic::arm_mve_vqrdmulh: + case Intrinsic::arm_mve_qrdmulh_predicated: + case Intrinsic::arm_mve_vqdmull: + case Intrinsic::arm_mve_vqdmull_predicated: + return true; + case Intrinsic::arm_mve_sub_predicated: + case Intrinsic::arm_mve_qsub_predicated: + case Intrinsic::arm_mve_vhsub: + case Intrinsic::arm_mve_hsub_predicated: + return N->getOperand(2).getNode() == Op; + default: + return false; + } + default: + return false; + } +} + // If this is a case we can't handle, return null and let the default // expansion code take care of it. SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, @@ -7720,6 +7793,20 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, if (SplatUndef.isAllOnes()) return DAG.getUNDEF(VT); + // If all the users of this constant splat are qr instruction variants, + // generate a vdup of the constant. + if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize && + (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) && + all_of(BVN->uses(), + [BVN](const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) { + EVT DupVT = SplatBitSize == 32 ? MVT::v4i32 + : SplatBitSize == 16 ? MVT::v8i16 + : MVT::v16i8; + SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32); + SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const); + return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup); + } + if ((ST->hasNEON() && SplatBitSize <= 64) || (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) { // Check if an immediate VMOV works. @@ -8313,9 +8400,8 @@ static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, SDLoc DL(Op); SmallVector<SDValue, 8> VTBLMask; - for (ArrayRef<int>::iterator - I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I) - VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32)); + for (int I : ShuffleMask) + VTBLMask.push_back(DAG.getConstant(I, DL, MVT::i32)); if (V2.getNode()->isUndef()) return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1, @@ -8346,6 +8432,8 @@ static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { static EVT getVectorTyFromPredicateVector(EVT VT) { switch (VT.getSimpleVT().SimpleTy) { + case MVT::v2i1: + return MVT::v2f64; case MVT::v4i1: return MVT::v4i32; case MVT::v8i1: @@ -8427,7 +8515,14 @@ static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, DAG.getUNDEF(NewVT), ShuffleMask); // Now return the result of comparing the shuffled vector with zero, - // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. + // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1 + // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s. + if (VT == MVT::v2i1) { + SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Shuffled); + SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC, + DAG.getConstant(ARMCC::NE, dl, MVT::i32)); + return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp); + } return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled, DAG.getConstant(ARMCC::NE, dl, MVT::i32)); } @@ -8927,8 +9022,15 @@ static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, ConVec = ExtractInto(NewV1, ConVec, j); ConVec = ExtractInto(NewV2, ConVec, j); - // Now return the result of comparing the subvector with zero, - // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. + // Now return the result of comparing the subvector with zero, which will + // generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1 we + // convert to a v4i1 compare to fill in the two halves of the i64 as i32s. + if (VT == MVT::v2i1) { + SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, ConVec); + SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC, + DAG.getConstant(ARMCC::NE, dl, MVT::i32)); + return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp); + } return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec, DAG.getConstant(ARMCC::NE, dl, MVT::i32)); }; @@ -8993,6 +9095,22 @@ static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT(); + if (NumElts == 2) { + EVT SubVT = MVT::v4i32; + SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT); + for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) { + SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1, + DAG.getIntPtrConstant(i, dl)); + SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt, + DAG.getConstant(j, dl, MVT::i32)); + SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt, + DAG.getConstant(j + 1, dl, MVT::i32)); + } + SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, SubVec, + DAG.getConstant(ARMCC::NE, dl, MVT::i32)); + return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp); + } + EVT SubVT = MVT::getVectorVT(ElType, NumElts); SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT); for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) { @@ -9839,16 +9957,17 @@ void ARMTargetLowering::ExpandDIV_Windows( static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) { LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); EVT MemVT = LD->getMemoryVT(); - assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && + assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || + MemVT == MVT::v16i1) && "Expected a predicate type!"); assert(MemVT == Op.getValueType()); assert(LD->getExtensionType() == ISD::NON_EXTLOAD && "Expected a non-extending load"); assert(LD->isUnindexed() && "Expected a unindexed load"); - // The basic MVE VLDR on a v4i1/v8i1 actually loads the entire 16bit + // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We - // need to make sure that 8/4 bits are actually loaded into the correct + // need to make sure that 8/4/2 bits are actually loaded into the correct // place, which means loading the value and then shuffling the values into // the bottom bits of the predicate. // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect @@ -9895,14 +10014,15 @@ void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results, static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) { StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); EVT MemVT = ST->getMemoryVT(); - assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && + assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || + MemVT == MVT::v16i1) && "Expected a predicate type!"); assert(MemVT == ST->getValue().getValueType()); assert(!ST->isTruncatingStore() && "Expected a non-extending store"); assert(ST->isUnindexed() && "Expected a unindexed store"); - // Only store the v4i1 or v8i1 worth of bits, via a buildvector with top bits - // unset and a scalar store. + // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with + // top bits unset and a scalar store. SDLoc dl(Op); SDValue Build = ST->getValue(); if (MemVT != MVT::v16i1) { @@ -9953,7 +10073,7 @@ static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, {ST->getChain(), Lo, Hi, ST->getBasePtr()}, MemVT, ST->getMemOperand()); } else if (Subtarget->hasMVEIntegerOps() && - ((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || + ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1))) { return LowerPredicateStore(Op, DAG); } @@ -10561,25 +10681,23 @@ void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI, // associated with. DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad; unsigned MaxCSNum = 0; - for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E; - ++BB) { - if (!BB->isEHPad()) continue; + for (MachineBasicBlock &BB : *MF) { + if (!BB.isEHPad()) + continue; // FIXME: We should assert that the EH_LABEL is the first MI in the landing // pad. - for (MachineBasicBlock::iterator - II = BB->begin(), IE = BB->end(); II != IE; ++II) { - if (!II->isEHLabel()) continue; + for (MachineInstr &II : BB) { + if (!II.isEHLabel()) + continue; - MCSymbol *Sym = II->getOperand(0).getMCSymbol(); + MCSymbol *Sym = II.getOperand(0).getMCSymbol(); if (!MF->hasCallSiteLandingPad(Sym)) continue; SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym); - for (SmallVectorImpl<unsigned>::iterator - CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end(); - CSI != CSE; ++CSI) { - CallSiteNumToLPad[*CSI].push_back(&*BB); - MaxCSNum = std::max(MaxCSNum, *CSI); + for (unsigned Idx : CallSiteIdxs) { + CallSiteNumToLPad[Idx].push_back(&BB); + MaxCSNum = std::max(MaxCSNum, Idx); } break; } @@ -14002,8 +14120,8 @@ static SDValue PerformANDCombine(SDNode *N, EVT VT = N->getValueType(0); SelectionDAG &DAG = DCI.DAG; - if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v4i1 || - VT == MVT::v8i1 || VT == MVT::v16i1) + if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 || + VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1) return SDValue(); APInt SplatBits, SplatUndef; @@ -14298,8 +14416,8 @@ static SDValue PerformORCombine(SDNode *N, if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) return SDValue(); - if (Subtarget->hasMVEIntegerOps() && - (VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)) + if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 || + VT == MVT::v8i1 || VT == MVT::v16i1)) return PerformORCombine_i1(N, DAG, Subtarget); APInt SplatBits, SplatUndef; @@ -14569,6 +14687,15 @@ static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) { if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(Cmp->getOperand(1))) return SDValue(); SDValue CSInc = Cmp->getOperand(0); + + // Ignore any `And 1` nodes that may not yet have been removed. We are + // looking for a value that produces 1/0, so these have no effect on the + // code. + while (CSInc.getOpcode() == ISD::AND && + isa<ConstantSDNode>(CSInc.getOperand(1)) && + CSInc.getConstantOperandVal(1) == 1 && CSInc->hasOneUse()) + CSInc = CSInc.getOperand(0); + if (CSInc.getOpcode() != ARMISD::CSINC || !isNullConstant(CSInc.getOperand(0)) || !isNullConstant(CSInc.getOperand(1)) || !CSInc->hasOneUse()) @@ -17897,6 +18024,23 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const { if (!VT.isInteger()) return SDValue(); + // Fold away an unneccessary CMPZ/CMOV + // CMOV A, B, C1, $cpsr, (CMPZ (CMOV 1, 0, C2, D), 0) -> + // if C1==EQ -> CMOV A, B, C2, $cpsr, D + // if C1==NE -> CMOV A, B, NOT(C2), $cpsr, D + if (N->getConstantOperandVal(2) == ARMCC::EQ || + N->getConstantOperandVal(2) == ARMCC::NE) { + ARMCC::CondCodes Cond; + if (SDValue C = IsCMPZCSINC(N->getOperand(4).getNode(), Cond)) { + if (N->getConstantOperandVal(2) == ARMCC::NE) + Cond = ARMCC::getOppositeCondition(Cond); + return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0), + N->getOperand(1), + DAG.getTargetConstant(Cond, SDLoc(N), MVT::i32), + N->getOperand(3), C); + } + } + // Materialize a boolean comparison for integers so we can avoid branching. if (isNullConstant(FalseVal)) { if (CC == ARMCC::EQ && isOneConstant(TrueVal)) { @@ -18564,7 +18708,8 @@ bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, return false; // These are for predicates - if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1)) { + if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 || + Ty == MVT::v2i1)) { if (Fast) *Fast = true; return true; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index e3b422358cae..1c5f8389f57c 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -69,6 +69,7 @@ class VectorType; CALL_PRED, // Function call that's predicable. CALL_NOLINK, // Function call with branch not branch-and-link. tSECALL, // CMSE non-secure function call. + t2CALL_BTI, // Thumb function call followed by BTI instruction. BRCOND, // Conditional branch. BR_JT, // Jumptable branch. BR2_JT, // Jumptable branch (2 level - jumptable entry is a jump). diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td index f53814a80e01..1ae0354ffc37 100644 --- a/llvm/lib/Target/ARM/ARMInstrMVE.td +++ b/llvm/lib/Target/ARM/ARMInstrMVE.td @@ -254,13 +254,6 @@ class MVEVectorVTInfo<ValueType vec, ValueType dblvec, // An LLVM ValueType representing a corresponding vector of // predicate bits, for use in ISel patterns that handle an IR // intrinsic describing the predicated form of the instruction. - // - // Usually, for a vector of N things, this will be vNi1. But for - // vectors of 2 values, we make an exception, and use v4i1 instead - // of v2i1. Rationale: MVE codegen doesn't support doing all the - // auxiliary operations on v2i1 (vector shuffles etc), and also, - // there's no MVE compare instruction that will _generate_ v2i1 - // directly. ValueType Pred = pred; // Same as Pred but for DblVec rather than Vec. @@ -294,25 +287,25 @@ class MVEVectorVTInfo<ValueType vec, ValueType dblvec, // Integer vector types that don't treat signed and unsigned differently. def MVE_v16i8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "i", ?>; def MVE_v8i16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "i", ?>; -def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "i", ?>; -def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "i", ?>; +def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "i", ?>; +def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "i", ?>; // Explicitly signed and unsigned integer vectors. They map to the // same set of LLVM ValueTypes as above, but are represented // differently in assembly and instruction encodings. def MVE_v16s8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "s", 0b0>; def MVE_v8s16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "s", 0b0>; -def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "s", 0b0>; -def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "s", 0b0>; +def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "s", 0b0>; +def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "s", 0b0>; def MVE_v16u8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "u", 0b1>; def MVE_v8u16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "u", 0b1>; -def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "u", 0b1>; -def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "u", 0b1>; +def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "u", 0b1>; +def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "u", 0b1>; // FP vector types. def MVE_v8f16 : MVEVectorVTInfo<v8f16, v4f32, v8i1, v4i1, 0b01, "f", ?>; -def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1, v4i1, 0b10, "f", ?>; -def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?, v4i1, ?, 0b11, "f", ?>; +def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1, v2i1, 0b10, "f", ?>; +def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?, v2i1, ?, 0b11, "f", ?>; // Polynomial vector types. def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>; @@ -2260,6 +2253,31 @@ let Predicates = [HasMVEInt] in { (v4i32 (ARMvmovImm (i32 1)))), (i32 1))), (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>; + + def : Pat<(v16i8 (ARMvshrsImm (addnsw (addnsw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)), + (v16i8 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDs8 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v8i16 (ARMvshrsImm (addnsw (addnsw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)), + (v8i16 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDs16 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v4i32 (ARMvshrsImm (addnsw (addnsw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)), + (v4i32 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDs32 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v16i8 (ARMvshruImm (addnuw (addnuw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)), + (v16i8 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDu8 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v8i16 (ARMvshruImm (addnuw (addnuw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)), + (v8i16 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDu16 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v4i32 (ARMvshruImm (addnuw (addnuw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)), + (v4i32 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>; } @@ -4450,6 +4468,11 @@ multiclass two_predops<SDPatternOperator opnode, Instruction insn> { (insn (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p1), rGPR)), (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p2), rGPR))), VCCR))>; + def v2i1 : Pat<(v2i1 (opnode (v2i1 VCCR:$p1), (v2i1 VCCR:$p2))), + (v2i1 (COPY_TO_REGCLASS + (insn (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p1), rGPR)), + (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p2), rGPR))), + VCCR))>; } let Predicates = [HasMVEInt] in { @@ -4469,20 +4492,20 @@ def load_align4 : PatFrag<(ops node:$ptr), (load node:$ptr), [{ }]>; let Predicates = [HasMVEInt] in { - foreach VT = [ v4i1, v8i1, v16i1 ] in { + foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in { def : Pat<(i32 (predicate_cast (VT VCCR:$src))), (i32 (COPY_TO_REGCLASS (VT VCCR:$src), VCCR))>; def : Pat<(VT (predicate_cast (i32 VCCR:$src))), (VT (COPY_TO_REGCLASS (i32 VCCR:$src), VCCR))>; - foreach VT2 = [ v4i1, v8i1, v16i1 ] in + foreach VT2 = [ v2i1, v4i1, v8i1, v16i1 ] in def : Pat<(VT (predicate_cast (VT2 VCCR:$src))), (VT (COPY_TO_REGCLASS (VT2 VCCR:$src), VCCR))>; } // If we happen to be casting from a load we can convert that straight // into a predicate load, so long as the load is of the correct type. - foreach VT = [ v4i1, v8i1, v16i1 ] in { + foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in { def : Pat<(VT (predicate_cast (i32 (load_align4 taddrmode_imm7<2>:$addr)))), (VT (VLDR_P0_off taddrmode_imm7<2>:$addr))>; } @@ -5350,33 +5373,40 @@ class MVE_VxADDSUB_qr<string iname, string suffix, } multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract, - Intrinsic unpred_int, Intrinsic pred_int> { + Intrinsic unpred_int, Intrinsic pred_int, PatFrag add_op, + SDNode shift_op> { def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract, VTI.Size>; defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI, unpred_int, pred_int, 1, 1>; + defvar Inst = !cast<Instruction>(NAME); + + let Predicates = [HasMVEInt] in { + def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))), (i32 1))), + (Inst MQPR:$Qm, rGPR:$Rn)>; + } } -multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI> : - MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd, - int_arm_mve_hadd_predicated>; +multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> : + MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, + add_op, shift_op>; -multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI> : - MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub, - int_arm_mve_hsub_predicated>; +multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> : + MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, + add_op, shift_op>; -defm MVE_VHADD_qr_s8 : MVE_VHADD_qr_m<MVE_v16s8>; -defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16>; -defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32>; -defm MVE_VHADD_qr_u8 : MVE_VHADD_qr_m<MVE_v16u8>; -defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16>; -defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32>; +defm MVE_VHADD_qr_s8 : MVE_VHADD_qr_m<MVE_v16s8, addnsw, ARMvshrsImm>; +defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16, addnsw, ARMvshrsImm>; +defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32, addnsw, ARMvshrsImm>; +defm MVE_VHADD_qr_u8 : MVE_VHADD_qr_m<MVE_v16u8, addnuw, ARMvshruImm>; +defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16, addnuw, ARMvshruImm>; +defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32, addnuw, ARMvshruImm>; -defm MVE_VHSUB_qr_s8 : MVE_VHSUB_qr_m<MVE_v16s8>; -defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16>; -defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32>; -defm MVE_VHSUB_qr_u8 : MVE_VHSUB_qr_m<MVE_v16u8>; -defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16>; -defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32>; +defm MVE_VHSUB_qr_s8 : MVE_VHSUB_qr_m<MVE_v16s8, subnsw, ARMvshrsImm>; +defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16, subnsw, ARMvshrsImm>; +defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32, subnsw, ARMvshrsImm>; +defm MVE_VHSUB_qr_u8 : MVE_VHSUB_qr_m<MVE_v16u8, subnuw, ARMvshruImm>; +defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16, subnuw, ARMvshruImm>; +defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32, subnuw, ARMvshruImm>; multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract, SDNode Op, Intrinsic PredInt, SDPatternOperator IdentityVec> { @@ -6778,11 +6808,15 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))), (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (vselect (v2i1 VCCR:$pred), (v2i64 MQPR:$v1), (v2i64 MQPR:$v2))), + (v2i64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))), (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))), (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2f64 (vselect (v2i1 VCCR:$pred), (v2f64 MQPR:$v1), (v2f64 MQPR:$v2))), + (v2f64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v16i8 (vselect (v16i8 MQPR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))), (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, @@ -6808,6 +6842,8 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (zext (v4i1 VCCR:$pred))), (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (zext (v2i1 VCCR:$pred))), + (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v16i8 (sext (v16i1 VCCR:$pred))), (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>; @@ -6815,6 +6851,8 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (sext (v4i1 VCCR:$pred))), (v4i32 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (sext (v2i1 VCCR:$pred))), + (v2i64 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v16i8 (anyext (v16i1 VCCR:$pred))), (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>; @@ -6822,6 +6860,8 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (anyext (v4i1 VCCR:$pred))), (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (anyext (v2i1 VCCR:$pred))), + (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; } let Predicates = [HasMVEFloat] in { @@ -6862,6 +6902,8 @@ def MVE_VPNOT : MVE_p<(outs VCCR:$P0), (ins VCCR:$P0_in), NoItinerary, } let Predicates = [HasMVEInt] in { + def : Pat<(v2i1 (xor (v2i1 VCCR:$pred), (v2i1 (predicate_cast (i32 65535))))), + (v2i1 (MVE_VPNOT (v2i1 VCCR:$pred)))>; def : Pat<(v4i1 (xor (v4i1 VCCR:$pred), (v4i1 (predicate_cast (i32 65535))))), (v4i1 (MVE_VPNOT (v4i1 VCCR:$pred)))>; def : Pat<(v8i1 (xor (v8i1 VCCR:$pred), (v8i1 (predicate_cast (i32 65535))))), diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td index 4471317f4ea4..6e8e61ca2b8e 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb2.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -5736,3 +5736,10 @@ def t2BTI : PACBTIHintSpaceNoOpsInst<"bti", 0b00001111>; def t2AUT : PACBTIHintSpaceUseInst<"aut", 0b00101101> { let hasSideEffects = 1; } + +def ARMt2CallBTI : SDNode<"ARMISD::t2CALL_BTI", SDT_ARMcall, + [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, SDNPVariadic]>; + +def t2CALL_BTI : PseudoInst<(outs), (ins pred:$p, thumb_bl_target:$func), + IIC_Br, [(ARMt2CallBTI tglobaladdr:$func)]>, + Requires<[IsThumb2]>, Sched<[WriteBrL]>; diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td index 9d1bfa414dff..dc5f1b92a6c2 100644 --- a/llvm/lib/Target/ARM/ARMInstrVFP.td +++ b/llvm/lib/Target/ARM/ARMInstrVFP.td @@ -1076,6 +1076,9 @@ multiclass vrint_inst_anpm<string opc, bits<2> rm, } } + def : InstAlias<!strconcat("vrint", opc, ".f16.f16\t$Sd, $Sm"), + (!cast<Instruction>(NAME#"H") HPR:$Sd, HPR:$Sm), 0>, + Requires<[HasFullFP16]>; def : InstAlias<!strconcat("vrint", opc, ".f32.f32\t$Sd, $Sm"), (!cast<Instruction>(NAME#"S") SPR:$Sd, SPR:$Sm), 0>, Requires<[HasFPARMv8]>; diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp index 3b10c60a0654..ef5fc12feb54 100644 --- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp +++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp @@ -2121,7 +2121,7 @@ bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { bool Modified = false; for (MachineBasicBlock &MBB : Fn) { Modified |= LoadStoreMultipleOpti(MBB); - if (STI->hasV5TOps()) + if (STI->hasV5TOps() && !AFI->shouldSignReturnAddress()) Modified |= MergeReturnIntoLDM(MBB); if (isThumb1) Modified |= CombineMovBx(MBB); @@ -2349,9 +2349,8 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB, unsigned LastOpcode = 0; unsigned LastBytes = 0; unsigned NumMove = 0; - for (int i = Ops.size() - 1; i >= 0; --i) { + for (MachineInstr *Op : llvm::reverse(Ops)) { // Make sure each operation has the same kind. - MachineInstr *Op = Ops[i]; unsigned LSMOpcode = getLoadStoreMultipleOpcode(Op->getOpcode(), ARM_AM::ia); if (LastOpcode && LSMOpcode != LastOpcode) diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp index 3874db5792d6..f822672c4477 100644 --- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp +++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp @@ -251,10 +251,7 @@ namespace { SetVector<MachineInstr *> &Predicates = PredicatedInsts[MI]->Predicates; if (Exclusive && Predicates.size() != 1) return false; - for (auto *PredMI : Predicates) - if (isVCTP(PredMI)) - return true; - return false; + return llvm::any_of(Predicates, isVCTP); } // Is the VPST, controlling the block entry, predicated upon a VCTP. @@ -351,10 +348,7 @@ namespace { } bool containsVCTP() const { - for (auto *MI : Insts) - if (isVCTP(MI)) - return true; - return false; + return llvm::any_of(Insts, isVCTP); } unsigned size() const { return Insts.size(); } @@ -1334,8 +1328,8 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) { bool Changed = false; // Process inner loops first. - for (auto I = ML->begin(), E = ML->end(); I != E; ++I) - Changed |= ProcessLoop(*I); + for (MachineLoop *L : *ML) + Changed |= ProcessLoop(L); LLVM_DEBUG({ dbgs() << "ARM Loops: Processing loop containing:\n"; @@ -1699,7 +1693,7 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) { // If any of the instructions between the VCMP and VPST are predicated // then a different code path is expected to have merged the VCMP and // VPST already. - if (!std::any_of(++MachineBasicBlock::iterator(VCMP), + if (std::none_of(++MachineBasicBlock::iterator(VCMP), MachineBasicBlock::iterator(VPST), hasVPRUse) && RDA->hasSameReachingDef(VCMP, VPST, VCMP->getOperand(1).getReg()) && RDA->hasSameReachingDef(VCMP, VPST, VCMP->getOperand(2).getReg())) { diff --git a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h index 4077fc058217..d8d937055d23 100644 --- a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h +++ b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h @@ -289,7 +289,7 @@ public: return false; if (SignReturnAddressAll) return true; - return LRSpilled; + return SpillsLR; } bool branchTargetEnforcement() const { return BranchTargetEnforcement; } diff --git a/llvm/lib/Target/ARM/ARMRegisterInfo.td b/llvm/lib/Target/ARM/ARMRegisterInfo.td index 760a5a5a20cf..194d65cad8d1 100644 --- a/llvm/lib/Target/ARM/ARMRegisterInfo.td +++ b/llvm/lib/Target/ARM/ARMRegisterInfo.td @@ -211,6 +211,8 @@ def FPCXTS : ARMReg<15, "fpcxts">; def ZR : ARMReg<15, "zr">, DwarfRegNum<[15]>; +def RA_AUTH_CODE : ARMReg<12, "ra_auth_code">, DwarfRegNum<[143]>; + // Register classes. // // pc == Program Counter @@ -395,7 +397,7 @@ def CCR : RegisterClass<"ARM", [i32], 32, (add CPSR)> { } // MVE Condition code register. -def VCCR : RegisterClass<"ARM", [i32, v16i1, v8i1, v4i1], 32, (add VPR)> { +def VCCR : RegisterClass<"ARM", [i32, v16i1, v8i1, v4i1, v2i1], 32, (add VPR)> { // let CopyCost = -1; // Don't allow copying of status registers. } diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h index d51a888c951f..e61b90af31b0 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.h +++ b/llvm/lib/Target/ARM/ARMSubtarget.h @@ -18,6 +18,7 @@ #include "ARMConstantPoolValue.h" #include "ARMFrameLowering.h" #include "ARMISelLowering.h" +#include "ARMMachineFunctionInfo.h" #include "ARMSelectionDAGInfo.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -534,6 +535,10 @@ protected: /// Selected instruction itineraries (one entry per itinerary class.) InstrItineraryData InstrItins; + /// NoBTIAtReturnTwice - Don't place a BTI instruction after + /// return-twice constructs (setjmp) + bool NoBTIAtReturnTwice = false; + /// Options passed via command line that could influence the target const TargetOptions &Options; @@ -840,6 +845,8 @@ public: /// to lr. This is always required on Thumb1-only targets, as the push and /// pop instructions can't access the high registers. bool splitFramePushPop(const MachineFunction &MF) const { + if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress()) + return true; return (getFramePointerReg() == ARM::R7 && MF.getTarget().Options.DisableFramePointerElim(MF)) || isThumb1Only(); @@ -948,6 +955,8 @@ public: bool hardenSlsRetBr() const { return HardenSlsRetBr; } bool hardenSlsBlr() const { return HardenSlsBlr; } bool hardenSlsNoComdat() const { return HardenSlsNoComdat; } + + bool getNoBTIAtReturnTwice() const { return NoBTIAtReturnTwice; } }; } // end namespace llvm diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 39f407ba7149..bfe078b06861 100644 --- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -137,21 +137,18 @@ public: int getFPReg() const { return FPReg; } void emitFnStartLocNotes() const { - for (Locs::const_iterator FI = FnStartLocs.begin(), FE = FnStartLocs.end(); - FI != FE; ++FI) - Parser.Note(*FI, ".fnstart was specified here"); + for (const SMLoc &Loc : FnStartLocs) + Parser.Note(Loc, ".fnstart was specified here"); } void emitCantUnwindLocNotes() const { - for (Locs::const_iterator UI = CantUnwindLocs.begin(), - UE = CantUnwindLocs.end(); UI != UE; ++UI) - Parser.Note(*UI, ".cantunwind was specified here"); + for (const SMLoc &Loc : CantUnwindLocs) + Parser.Note(Loc, ".cantunwind was specified here"); } void emitHandlerDataLocNotes() const { - for (Locs::const_iterator HI = HandlerDataLocs.begin(), - HE = HandlerDataLocs.end(); HI != HE; ++HI) - Parser.Note(*HI, ".handlerdata was specified here"); + for (const SMLoc &Loc : HandlerDataLocs) + Parser.Note(Loc, ".handlerdata was specified here"); } void emitPersonalityLocNotes() const { @@ -452,7 +449,8 @@ class ARMAsmParser : public MCTargetAsmParser { int tryParseRegister(); bool tryParseRegisterWithWriteBack(OperandVector &); int tryParseShiftRegister(OperandVector &); - bool parseRegisterList(OperandVector &, bool EnforceOrder = true); + bool parseRegisterList(OperandVector &, bool EnforceOrder = true, + bool AllowRAAC = false); bool parseMemory(OperandVector &); bool parseOperand(OperandVector &, StringRef Mnemonic); bool parsePrefix(ARMMCExpr::VariantKind &RefKind); @@ -2572,17 +2570,15 @@ public: void addRegListOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); const SmallVectorImpl<unsigned> &RegList = getRegList(); - for (SmallVectorImpl<unsigned>::const_iterator - I = RegList.begin(), E = RegList.end(); I != E; ++I) - Inst.addOperand(MCOperand::createReg(*I)); + for (unsigned Reg : RegList) + Inst.addOperand(MCOperand::createReg(Reg)); } void addRegListWithAPSROperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); const SmallVectorImpl<unsigned> &RegList = getRegList(); - for (SmallVectorImpl<unsigned>::const_iterator - I = RegList.begin(), E = RegList.end(); I != E; ++I) - Inst.addOperand(MCOperand::createReg(*I)); + for (unsigned Reg : RegList) + Inst.addOperand(MCOperand::createReg(Reg)); } void addDPRRegListOperands(MCInst &Inst, unsigned N) const { @@ -4464,8 +4460,8 @@ insertNoDuplicates(SmallVectorImpl<std::pair<unsigned, unsigned>> &Regs, } /// Parse a register list. -bool ARMAsmParser::parseRegisterList(OperandVector &Operands, - bool EnforceOrder) { +bool ARMAsmParser::parseRegisterList(OperandVector &Operands, bool EnforceOrder, + bool AllowRAAC) { MCAsmParser &Parser = getParser(); if (Parser.getTok().isNot(AsmToken::LCurly)) return TokError("Token is not a Left Curly Brace"); @@ -4478,7 +4474,8 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, int Reg = tryParseRegister(); if (Reg == -1) return Error(RegLoc, "register expected"); - + if (!AllowRAAC && Reg == ARM::RA_AUTH_CODE) + return Error(RegLoc, "pseudo-register not allowed"); // The reglist instructions have at most 16 registers, so reserve // space for that many. int EReg = 0; @@ -4492,7 +4489,8 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, ++Reg; } const MCRegisterClass *RC; - if (ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg)) + if (Reg == ARM::RA_AUTH_CODE || + ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg)) RC = &ARMMCRegisterClasses[ARM::GPRRegClassID]; else if (ARMMCRegisterClasses[ARM::DPRRegClassID].contains(Reg)) RC = &ARMMCRegisterClasses[ARM::DPRRegClassID]; @@ -4513,11 +4511,15 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, while (Parser.getTok().is(AsmToken::Comma) || Parser.getTok().is(AsmToken::Minus)) { if (Parser.getTok().is(AsmToken::Minus)) { + if (Reg == ARM::RA_AUTH_CODE) + return Error(RegLoc, "pseudo-register not allowed"); Parser.Lex(); // Eat the minus. SMLoc AfterMinusLoc = Parser.getTok().getLoc(); int EndReg = tryParseRegister(); if (EndReg == -1) return Error(AfterMinusLoc, "register expected"); + if (EndReg == ARM::RA_AUTH_CODE) + return Error(AfterMinusLoc, "pseudo-register not allowed"); // Allow Q regs and just interpret them as the two D sub-registers. if (ARMMCRegisterClasses[ARM::QPRRegClassID].contains(EndReg)) EndReg = getDRegFromQReg(EndReg) + 1; @@ -4526,7 +4528,9 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, if (Reg == EndReg) continue; // The register must be in the same register class as the first. - if (!RC->contains(EndReg)) + if ((Reg == ARM::RA_AUTH_CODE && + RC != &ARMMCRegisterClasses[ARM::GPRRegClassID]) || + (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg))) return Error(AfterMinusLoc, "invalid register in register list"); // Ranges must go from low to high. if (MRI->getEncodingValue(Reg) > MRI->getEncodingValue(EndReg)) @@ -4551,13 +4555,15 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, Reg = tryParseRegister(); if (Reg == -1) return Error(RegLoc, "register expected"); + if (!AllowRAAC && Reg == ARM::RA_AUTH_CODE) + return Error(RegLoc, "pseudo-register not allowed"); // Allow Q regs and just interpret them as the two D sub-registers. bool isQReg = false; if (ARMMCRegisterClasses[ARM::QPRRegClassID].contains(Reg)) { Reg = getDRegFromQReg(Reg); isQReg = true; } - if (!RC->contains(Reg) && + if (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg) && RC->getID() == ARMMCRegisterClasses[ARM::GPRRegClassID].getID() && ARMMCRegisterClasses[ARM::GPRwithAPSRnospRegClassID].contains(Reg)) { // switch the register classes, as GPRwithAPSRnospRegClassID is a partial @@ -4577,7 +4583,9 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, continue; } // The register must be in the same register class as the first. - if (!RC->contains(Reg)) + if ((Reg == ARM::RA_AUTH_CODE && + RC != &ARMMCRegisterClasses[ARM::GPRRegClassID]) || + (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg))) return Error(RegLoc, "invalid register in register list"); // In most cases, the list must be monotonically increasing. An // exception is CLRM, which is order-independent anyway, so @@ -7106,13 +7114,12 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, return Error(Loc, "too many conditions on VPT instruction"); } unsigned Mask = 8; - for (unsigned i = ITMask.size(); i != 0; --i) { - char pos = ITMask[i - 1]; - if (pos != 't' && pos != 'e') { + for (char Pos : llvm::reverse(ITMask)) { + if (Pos != 't' && Pos != 'e') { return Error(Loc, "illegal IT block condition mask '" + ITMask + "'"); } Mask >>= 1; - if (ITMask[i - 1] == 'e') + if (Pos == 'e') Mask |= 8; } Operands.push_back(ARMOperand::CreateITMask(Mask, Loc)); @@ -11685,7 +11692,7 @@ bool ARMAsmParser::parseDirectiveRegSave(SMLoc L, bool IsVector) { SmallVector<std::unique_ptr<MCParsedAsmOperand>, 1> Operands; // Parse the register list - if (parseRegisterList(Operands) || + if (parseRegisterList(Operands, true, true) || parseToken(AsmToken::EndOfStatement, "unexpected token in directive")) return true; ARMOperand &Op = (ARMOperand &)*Operands[0]; diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp index 896b104e8d97..e060e59e3759 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp @@ -1289,34 +1289,65 @@ void ARMELFStreamer::emitPad(int64_t Offset) { PendingOffset -= Offset; } -void ARMELFStreamer::emitRegSave(const SmallVectorImpl<unsigned> &RegList, - bool IsVector) { - // Collect the registers in the register list - unsigned Count = 0; +static std::pair<unsigned, unsigned> +collectHWRegs(const MCRegisterInfo &MRI, unsigned Idx, + const SmallVectorImpl<unsigned> &RegList, bool IsVector, + uint32_t &Mask_) { uint32_t Mask = 0; - const MCRegisterInfo *MRI = getContext().getRegisterInfo(); - for (size_t i = 0; i < RegList.size(); ++i) { - unsigned Reg = MRI->getEncodingValue(RegList[i]); + unsigned Count = 0; + while (Idx > 0) { + unsigned Reg = RegList[Idx - 1]; + if (Reg == ARM::RA_AUTH_CODE) + break; + Reg = MRI.getEncodingValue(Reg); assert(Reg < (IsVector ? 32U : 16U) && "Register out of range"); unsigned Bit = (1u << Reg); if ((Mask & Bit) == 0) { Mask |= Bit; ++Count; } + --Idx; } - // Track the change the $sp offset: For the .save directive, the - // corresponding push instruction will decrease the $sp by (4 * Count). - // For the .vsave directive, the corresponding vpush instruction will - // decrease $sp by (8 * Count). - SPOffset -= Count * (IsVector ? 8 : 4); + Mask_ = Mask; + return {Idx, Count}; +} - // Emit the opcode - FlushPendingOffset(); - if (IsVector) - UnwindOpAsm.EmitVFPRegSave(Mask); - else - UnwindOpAsm.EmitRegSave(Mask); +void ARMELFStreamer::emitRegSave(const SmallVectorImpl<unsigned> &RegList, + bool IsVector) { + uint32_t Mask; + unsigned Idx, Count; + const MCRegisterInfo &MRI = *getContext().getRegisterInfo(); + + // Collect the registers in the register list. Issue unwinding instructions in + // three parts: ordinary hardware registers, return address authentication + // code pseudo register, the rest of the registers. The RA PAC is kept in an + // architectural register (usually r12), but we treat it as a special case in + // order to distinguish between that register containing RA PAC or a general + // value. + Idx = RegList.size(); + while (Idx > 0) { + std::tie(Idx, Count) = collectHWRegs(MRI, Idx, RegList, IsVector, Mask); + if (Count) { + // Track the change the $sp offset: For the .save directive, the + // corresponding push instruction will decrease the $sp by (4 * Count). + // For the .vsave directive, the corresponding vpush instruction will + // decrease $sp by (8 * Count). + SPOffset -= Count * (IsVector ? 8 : 4); + + // Emit the opcode + FlushPendingOffset(); + if (IsVector) + UnwindOpAsm.EmitVFPRegSave(Mask); + else + UnwindOpAsm.EmitRegSave(Mask); + } else if (Idx > 0 && RegList[Idx - 1] == ARM::RA_AUTH_CODE) { + --Idx; + SPOffset -= 4; + FlushPendingOffset(); + UnwindOpAsm.EmitRegSave(0); + } + } } void ARMELFStreamer::emitUnwindRaw(int64_t Offset, diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp index 781627c3c425..50f416b23db2 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp @@ -64,8 +64,11 @@ namespace { } // end anonymous namespace void UnwindOpcodeAssembler::EmitRegSave(uint32_t RegSave) { - if (RegSave == 0u) + if (RegSave == 0u) { + // That's the special case for RA PAC. + EmitInt8(ARM::EHABI::UNWIND_OPCODE_POP_RA_AUTH_CODE); return; + } // One byte opcode to save register r14 and r11-r4 if (RegSave & (1u << 4)) { diff --git a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp index dc58b5427425..7e31ea77f4f5 100644 --- a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp +++ b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp @@ -366,7 +366,7 @@ bool MVETPAndVPTOptimisations::MergeLoopEnd(MachineLoop *ML) { while (!Worklist.empty()) { Register Reg = Worklist.pop_back_val(); for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) { - if (count(ExpectedUsers, &MI)) + if (llvm::is_contained(ExpectedUsers, &MI)) continue; if (MI.getOpcode() != TargetOpcode::COPY || !MI.getOperand(0).getReg().isVirtual()) { diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp index 6a5bc9284266..0e6960bce32b 100644 --- a/llvm/lib/Target/ARM/MVETailPredication.cpp +++ b/llvm/lib/Target/ARM/MVETailPredication.cpp @@ -213,7 +213,8 @@ bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask, auto *TC = SE->getSCEV(TripCount); int VectorWidth = cast<FixedVectorType>(ActiveLaneMask->getType())->getNumElements(); - if (VectorWidth != 4 && VectorWidth != 8 && VectorWidth != 16) + if (VectorWidth != 2 && VectorWidth != 4 && VectorWidth != 8 && + VectorWidth != 16) return false; ConstantInt *ConstElemCount = nullptr; @@ -371,15 +372,10 @@ void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask, switch (VectorWidth) { default: llvm_unreachable("unexpected number of lanes"); + case 2: VCTPID = Intrinsic::arm_mve_vctp64; break; case 4: VCTPID = Intrinsic::arm_mve_vctp32; break; case 8: VCTPID = Intrinsic::arm_mve_vctp16; break; case 16: VCTPID = Intrinsic::arm_mve_vctp8; break; - - // FIXME: vctp64 currently not supported because the predicate - // vector wants to be <2 x i1>, but v2i1 is not a legal MVE - // type, so problems happen at isel time. - // Intrinsic::arm_mve_vctp64 exists for ACLE intrinsics - // purposes, but takes a v4i1 instead of a v2i1. } Function *VCTP = Intrinsic::getDeclaration(M, VCTPID); Value *VCTPCall = Builder.CreateCall(VCTP, Processed); diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp index 224c61b9f065..54e80a095dd4 100644 --- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp +++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp @@ -824,8 +824,8 @@ bool Thumb1FrameLowering::spillCalleeSavedRegisters( ARMRegSet CopyRegs; // Registers which can be used after pushing // LoRegs for saving HiRegs. - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i-1].getReg(); + for (const CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); if (ARM::tGPRRegClass.contains(Reg) || Reg == ARM::LR) { LoRegsToSave[Reg] = true; @@ -1021,8 +1021,7 @@ bool Thumb1FrameLowering::restoreCalleeSavedRegisters( BuildMI(MF, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL)); bool NeedsPop = false; - for (unsigned i = CSI.size(); i != 0; --i) { - CalleeSavedInfo &Info = CSI[i-1]; + for (CalleeSavedInfo &Info : llvm::reverse(CSI)) { unsigned Reg = Info.getReg(); // High registers (excluding lr) have already been dealt with @@ -1067,7 +1066,7 @@ bool Thumb1FrameLowering::restoreCalleeSavedRegisters( if (NeedsPop) MBB.insert(MI, &*MIB); else - MF.DeleteMachineInstr(MIB); + MF.deleteMachineInstr(MIB); return true; } |