summaryrefslogtreecommitdiff
path: root/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/SystemZ/SystemZISelLowering.cpp')
-rw-r--r--llvm/lib/Target/SystemZ/SystemZISelLowering.cpp340
1 files changed, 227 insertions, 113 deletions
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 603446755aaf..d70d48638b14 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -285,10 +285,13 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
// Give LowerOperation the chance to replace 64-bit ORs with subregs.
setOperationAction(ISD::OR, MVT::i64, Custom);
- // FIXME: Can we support these natively?
+ // Expand 128 bit shifts without using a libcall.
setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
+ setLibcallName(RTLIB::SRL_I128, nullptr);
+ setLibcallName(RTLIB::SHL_I128, nullptr);
+ setLibcallName(RTLIB::SRA_I128, nullptr);
// We have native instructions for i8, i16 and i32 extensions, but not i1.
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
@@ -850,7 +853,7 @@ bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
}
bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
- EVT VT, unsigned, unsigned, MachineMemOperand::Flags, bool *Fast) const {
+ EVT VT, unsigned, Align, MachineMemOperand::Flags, bool *Fast) const {
// Unaligned accesses should never be slower than the expanded version.
// We check specifically for aligned accesses in the few cases where
// they are required.
@@ -1365,6 +1368,55 @@ static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
}
}
+static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
+ SDLoc DL(In);
+ SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
+ DAG.getIntPtrConstant(0, DL));
+ SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
+ DAG.getIntPtrConstant(1, DL));
+ SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL,
+ MVT::Untyped, Hi, Lo);
+ return SDValue(Pair, 0);
+}
+
+static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
+ SDLoc DL(In);
+ SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
+ DL, MVT::i64, In);
+ SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
+ DL, MVT::i64, In);
+ return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
+}
+
+bool SystemZTargetLowering::splitValueIntoRegisterParts(
+ SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
+ unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
+ EVT ValueVT = Val.getValueType();
+ assert((ValueVT != MVT::i128 ||
+ ((NumParts == 1 && PartVT == MVT::Untyped) ||
+ (NumParts == 2 && PartVT == MVT::i64))) &&
+ "Unknown handling of i128 value.");
+ if (ValueVT == MVT::i128 && NumParts == 1) {
+ // Inline assembly operand.
+ Parts[0] = lowerI128ToGR128(DAG, Val);
+ return true;
+ }
+ return false;
+}
+
+SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
+ SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
+ MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
+ assert((ValueVT != MVT::i128 ||
+ ((NumParts == 1 && PartVT == MVT::Untyped) ||
+ (NumParts == 2 && PartVT == MVT::i64))) &&
+ "Unknown handling of i128 value.");
+ if (ValueVT == MVT::i128 && NumParts == 1)
+ // Inline assembly operand.
+ return lowerGR128ToI128(DAG, Parts[0]);
+ return SDValue();
+}
+
SDValue SystemZTargetLowering::LowerFormalArguments(
SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
@@ -1482,20 +1534,20 @@ SDValue SystemZTargetLowering::LowerFormalArguments(
// ...and a similar frame index for the caller-allocated save area
// that will be used to store the incoming registers.
int64_t RegSaveOffset =
- -SystemZMC::CallFrameSize + TFL->getRegSpillOffset(MF, SystemZ::R2D) - 16;
+ -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, SystemZ::R2D) - 16;
unsigned RegSaveIndex = MFI.CreateFixedObject(1, RegSaveOffset, true);
FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
// Store the FPR varargs in the reserved frame slots. (We store the
// GPRs as part of the prologue.)
- if (NumFixedFPRs < SystemZ::NumArgFPRs && !useSoftFloat()) {
- SDValue MemOps[SystemZ::NumArgFPRs];
- for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) {
- unsigned Offset = TFL->getRegSpillOffset(MF, SystemZ::ArgFPRs[I]);
+ if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
+ SDValue MemOps[SystemZ::ELFNumArgFPRs];
+ for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
+ unsigned Offset = TFL->getRegSpillOffset(MF, SystemZ::ELFArgFPRs[I]);
int FI =
- MFI.CreateFixedObject(8, -SystemZMC::CallFrameSize + Offset, true);
+ MFI.CreateFixedObject(8, -SystemZMC::ELFCallFrameSize + Offset, true);
SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
- unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I],
+ unsigned VReg = MF.addLiveIn(SystemZ::ELFArgFPRs[I],
&SystemZ::FP64BitRegClass);
SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
@@ -1504,7 +1556,7 @@ SDValue SystemZTargetLowering::LowerFormalArguments(
// Join the stores, which are independent of one another.
Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
makeArrayRef(&MemOps[NumFixedFPRs],
- SystemZ::NumArgFPRs-NumFixedFPRs));
+ SystemZ::ELFNumArgFPRs-NumFixedFPRs));
}
}
@@ -1547,6 +1599,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
bool IsVarArg = CLI.IsVarArg;
MachineFunction &MF = DAG.getMachineFunction();
EVT PtrVT = getPointerTy(MF.getDataLayout());
+ LLVMContext &Ctx = *DAG.getContext();
// Detect unsupported vector argument and return types.
if (Subtarget.hasVector()) {
@@ -1556,7 +1609,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
// Analyze the operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
- SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
+ SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
// We don't support GuaranteedTailCallOpt, only automatically-detected
@@ -1581,14 +1634,25 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
if (VA.getLocInfo() == CCValAssign::Indirect) {
// Store the argument in a stack slot and pass its address.
- SDValue SpillSlot = DAG.CreateStackTemporary(Outs[I].ArgVT);
+ unsigned ArgIndex = Outs[I].OrigArgIndex;
+ EVT SlotVT;
+ if (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
+ // Allocate the full stack space for a promoted (and split) argument.
+ Type *OrigArgType = CLI.Args[Outs[I].OrigArgIndex].Ty;
+ EVT OrigArgVT = getValueType(MF.getDataLayout(), OrigArgType);
+ MVT PartVT = getRegisterTypeForCallingConv(Ctx, CLI.CallConv, OrigArgVT);
+ unsigned N = getNumRegistersForCallingConv(Ctx, CLI.CallConv, OrigArgVT);
+ SlotVT = EVT::getIntegerVT(Ctx, PartVT.getSizeInBits() * N);
+ } else {
+ SlotVT = Outs[I].ArgVT;
+ }
+ SDValue SpillSlot = DAG.CreateStackTemporary(SlotVT);
int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
MemOpChains.push_back(
DAG.getStore(Chain, DL, ArgValue, SpillSlot,
MachinePointerInfo::getFixedStack(MF, FI)));
// If the original argument was split (e.g. i128), we need
// to store all parts of it here (and pass just one address).
- unsigned ArgIndex = Outs[I].OrigArgIndex;
assert (Outs[I].PartOffset == 0);
while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
SDValue PartValue = OutVals[I + 1];
@@ -1598,6 +1662,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
MemOpChains.push_back(
DAG.getStore(Chain, DL, PartValue, Address,
MachinePointerInfo::getFixedStack(MF, FI)));
+ assert((PartOffset + PartValue.getValueType().getStoreSize() <=
+ SlotVT.getStoreSize()) && "Not enough space for argument part!");
++I;
}
ArgValue = SpillSlot;
@@ -1614,7 +1680,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
// floats are passed as right-justified 8-byte values.
if (!StackPtr.getNode())
StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT);
- unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset();
+ unsigned Offset = SystemZMC::ELFCallFrameSize + VA.getLocMemOffset();
if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
Offset += 4;
SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
@@ -1691,7 +1757,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
// Assign locations to each value returned by this call.
SmallVector<CCValAssign, 16> RetLocs;
- CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
+ CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ);
// Copy all of the result registers out of their specified physreg.
@@ -3264,12 +3330,10 @@ SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
EVT PtrVT = getPointerTy(DAG.getDataLayout());
- // Return null if the back chain is not present.
- bool HasBackChain = MF.getFunction().hasFnAttribute("backchain");
- if (TFL->usePackedStack(MF) && !HasBackChain)
- return DAG.getConstant(0, DL, PtrVT);
-
- // By definition, the frame address is the address of the back chain.
+ // By definition, the frame address is the address of the back chain. (In
+ // the case of packed stack without backchain, return the address where the
+ // backchain would have been stored. This will either be an unused space or
+ // contain a saved register).
int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
@@ -3868,7 +3932,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
Node->getMemOperand());
// We have to enforce sequential consistency by performing a
// serialization operation after the store.
- if (Node->getOrdering() == AtomicOrdering::SequentiallyConsistent)
+ if (Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
MVT::Other, Chain), 0);
return Chain;
@@ -4042,7 +4106,10 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
- DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
+ // emitAtomicCmpSwapW() will zero extend the result (original value).
+ SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
+ DAG.getValueType(NarrowVT));
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
return SDValue();
@@ -5471,27 +5538,6 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
// Lower operations with invalid operand or result types (currently used
// only for 128-bit integer types).
-
-static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
- SDLoc DL(In);
- SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
- DAG.getIntPtrConstant(0, DL));
- SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
- DAG.getIntPtrConstant(1, DL));
- SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL,
- MVT::Untyped, Hi, Lo);
- return SDValue(Pair, 0);
-}
-
-static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
- SDLoc DL(In);
- SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
- DL, MVT::i64, In);
- SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
- DL, MVT::i64, In);
- return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
-}
-
void
SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
SmallVectorImpl<SDValue> &Results,
@@ -5519,7 +5565,7 @@ SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
DL, Tys, Ops, MVT::i128, MMO);
// We have to enforce sequential consistency by performing a
// serialization operation after the store.
- if (cast<AtomicSDNode>(N)->getOrdering() ==
+ if (cast<AtomicSDNode>(N)->getSuccessOrdering() ==
AtomicOrdering::SequentiallyConsistent)
Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL,
MVT::Other, Res), 0);
@@ -7377,7 +7423,7 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
// StartMBB:
// ...
// %OrigVal = L Disp(%Base)
- // # fall through to LoopMMB
+ // # fall through to LoopMBB
MBB = StartMBB;
BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
MBB->addSuccessor(LoopMBB);
@@ -7389,7 +7435,7 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
// %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
// %Dest = CS %OldVal, %NewVal, Disp(%Base)
// JNE LoopMBB
- // # fall through to DoneMMB
+ // # fall through to DoneMBB
MBB = LoopMBB;
BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
.addReg(OrigVal).addMBB(StartMBB)
@@ -7497,7 +7543,7 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
// StartMBB:
// ...
// %OrigVal = L Disp(%Base)
- // # fall through to LoopMMB
+ // # fall through to LoopMBB
MBB = StartMBB;
BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
MBB->addSuccessor(LoopMBB);
@@ -7523,7 +7569,7 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
// UseAltMBB:
// %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
- // # fall through to UpdateMMB
+ // # fall through to UpdateMBB
MBB = UseAltMBB;
if (IsSubWord)
BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)
@@ -7537,7 +7583,7 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
// %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
// %Dest = CS %OldVal, %NewVal, Disp(%Base)
// JNE LoopMBB
- // # fall through to DoneMMB
+ // # fall through to DoneMBB
MBB = UpdateMBB;
BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)
.addReg(RotatedOldVal).addMBB(LoopMBB)
@@ -7564,7 +7610,6 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
MachineBasicBlock *
SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
MachineBasicBlock *MBB) const {
-
MachineFunction &MF = *MBB->getParent();
const SystemZInstrInfo *TII =
static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
@@ -7574,7 +7619,7 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
Register Dest = MI.getOperand(0).getReg();
MachineOperand Base = earlyUseOperand(MI.getOperand(1));
int64_t Disp = MI.getOperand(2).getImm();
- Register OrigCmpVal = MI.getOperand(3).getReg();
+ Register CmpVal = MI.getOperand(3).getReg();
Register OrigSwapVal = MI.getOperand(4).getReg();
Register BitShift = MI.getOperand(5).getReg();
Register NegBitShift = MI.getOperand(6).getReg();
@@ -7583,19 +7628,19 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
- // Get the right opcodes for the displacement.
+ // Get the right opcodes for the displacement and zero-extension.
unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
+ unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
assert(LOpcode && CSOpcode && "Displacement out of range");
// Create virtual registers for temporary results.
Register OrigOldVal = MRI.createVirtualRegister(RC);
Register OldVal = MRI.createVirtualRegister(RC);
- Register CmpVal = MRI.createVirtualRegister(RC);
Register SwapVal = MRI.createVirtualRegister(RC);
Register StoreVal = MRI.createVirtualRegister(RC);
+ Register OldValRot = MRI.createVirtualRegister(RC);
Register RetryOldVal = MRI.createVirtualRegister(RC);
- Register RetryCmpVal = MRI.createVirtualRegister(RC);
Register RetrySwapVal = MRI.createVirtualRegister(RC);
// Insert 2 basic blocks for the loop.
@@ -7607,7 +7652,7 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
// StartMBB:
// ...
// %OrigOldVal = L Disp(%Base)
- // # fall through to LoopMMB
+ // # fall through to LoopMBB
MBB = StartMBB;
BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal)
.add(Base)
@@ -7617,34 +7662,32 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
// LoopMBB:
// %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
- // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ]
// %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
- // %Dest = RLL %OldVal, BitSize(%BitShift)
+ // %OldValRot = RLL %OldVal, BitSize(%BitShift)
// ^^ The low BitSize bits contain the field
// of interest.
- // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0
+ // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
// ^^ Replace the upper 32-BitSize bits of the
- // comparison value with those that we loaded,
- // so that we can use a full word comparison.
- // CR %Dest, %RetryCmpVal
+ // swap value with those that we loaded and rotated.
+ // %Dest = LL[CH] %OldValRot
+ // CR %Dest, %CmpVal
// JNE DoneMBB
// # Fall through to SetMBB
MBB = LoopMBB;
BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
.addReg(OrigOldVal).addMBB(StartMBB)
.addReg(RetryOldVal).addMBB(SetMBB);
- BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal)
- .addReg(OrigCmpVal).addMBB(StartMBB)
- .addReg(RetryCmpVal).addMBB(SetMBB);
BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
.addReg(OrigSwapVal).addMBB(StartMBB)
.addReg(RetrySwapVal).addMBB(SetMBB);
- BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest)
+ BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)
.addReg(OldVal).addReg(BitShift).addImm(BitSize);
- BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal)
- .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
+ BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
+ .addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0);
+ BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)
+ .addReg(OldValRot);
BuildMI(MBB, DL, TII->get(SystemZ::CR))
- .addReg(Dest).addReg(RetryCmpVal);
+ .addReg(Dest).addReg(CmpVal);
BuildMI(MBB, DL, TII->get(SystemZ::BRC))
.addImm(SystemZ::CCMASK_ICMP)
.addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
@@ -7652,17 +7695,12 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
MBB->addSuccessor(SetMBB);
// SetMBB:
- // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0
- // ^^ Replace the upper 32-BitSize bits of the new
- // value with those that we loaded.
- // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
+ // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
// ^^ Rotate the new field to its proper position.
- // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base)
+ // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
// JNE LoopMBB
- // # fall through to ExitMMB
+ // # fall through to ExitMBB
MBB = SetMBB;
- BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
- .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
.addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
@@ -7757,43 +7795,99 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
uint64_t DestDisp = MI.getOperand(1).getImm();
MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2));
uint64_t SrcDisp = MI.getOperand(3).getImm();
- uint64_t Length = MI.getOperand(4).getImm();
+ MachineOperand &LengthMO = MI.getOperand(4);
+ uint64_t ImmLength = LengthMO.isImm() ? LengthMO.getImm() : 0;
+ Register LenMinus1Reg =
+ LengthMO.isReg() ? LengthMO.getReg() : SystemZ::NoRegister;
// When generating more than one CLC, all but the last will need to
// branch to the end when a difference is found.
- MachineBasicBlock *EndMBB = (Length > 256 && Opcode == SystemZ::CLC ?
- SystemZ::splitBlockAfter(MI, MBB) : nullptr);
+ MachineBasicBlock *EndMBB = (ImmLength > 256 && Opcode == SystemZ::CLC
+ ? SystemZ::splitBlockAfter(MI, MBB)
+ : nullptr);
// Check for the loop form, in which operand 5 is the trip count.
if (MI.getNumExplicitOperands() > 5) {
+ Register StartCountReg = MI.getOperand(5).getReg();
bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
- Register StartCountReg = MI.getOperand(5).getReg();
- Register StartSrcReg = forceReg(MI, SrcBase, TII);
- Register StartDestReg = (HaveSingleBase ? StartSrcReg :
- forceReg(MI, DestBase, TII));
+ auto loadZeroAddress = [&]() -> MachineOperand {
+ Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
+ BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0);
+ return MachineOperand::CreateReg(Reg, false);
+ };
+ if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
+ DestBase = loadZeroAddress();
+ if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
+ SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
+
+ MachineBasicBlock *StartMBB = nullptr;
+ MachineBasicBlock *LoopMBB = nullptr;
+ MachineBasicBlock *NextMBB = nullptr;
+ MachineBasicBlock *DoneMBB = nullptr;
+ MachineBasicBlock *AllDoneMBB = nullptr;
+
+ Register StartSrcReg = forceReg(MI, SrcBase, TII);
+ Register StartDestReg =
+ (HaveSingleBase ? StartSrcReg : forceReg(MI, DestBase, TII));
const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
Register ThisSrcReg = MRI.createVirtualRegister(RC);
- Register ThisDestReg = (HaveSingleBase ? ThisSrcReg :
- MRI.createVirtualRegister(RC));
+ Register ThisDestReg =
+ (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RC));
Register NextSrcReg = MRI.createVirtualRegister(RC);
- Register NextDestReg = (HaveSingleBase ? NextSrcReg :
- MRI.createVirtualRegister(RC));
-
+ Register NextDestReg =
+ (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RC));
RC = &SystemZ::GR64BitRegClass;
Register ThisCountReg = MRI.createVirtualRegister(RC);
Register NextCountReg = MRI.createVirtualRegister(RC);
- MachineBasicBlock *StartMBB = MBB;
- MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
- MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
- MachineBasicBlock *NextMBB =
- (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
+ if (LengthMO.isReg()) {
+ AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
+ StartMBB = SystemZ::emitBlockAfter(MBB);
+ LoopMBB = SystemZ::emitBlockAfter(StartMBB);
+ NextMBB = LoopMBB;
+ DoneMBB = SystemZ::emitBlockAfter(LoopMBB);
- // StartMBB:
- // # fall through to LoopMMB
- MBB->addSuccessor(LoopMBB);
+ // MBB:
+ // # Jump to AllDoneMBB if LenMinus1Reg is -1, or fall thru to StartMBB.
+ BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
+ .addReg(LenMinus1Reg).addImm(-1);
+ BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+ .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
+ .addMBB(AllDoneMBB);
+ MBB->addSuccessor(AllDoneMBB);
+ MBB->addSuccessor(StartMBB);
+
+ // StartMBB:
+ // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
+ MBB = StartMBB;
+ BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
+ .addReg(StartCountReg).addImm(0);
+ BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+ .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
+ .addMBB(DoneMBB);
+ MBB->addSuccessor(DoneMBB);
+ MBB->addSuccessor(LoopMBB);
+ }
+ else {
+ StartMBB = MBB;
+ DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
+ LoopMBB = SystemZ::emitBlockAfter(StartMBB);
+ NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
+
+ // StartMBB:
+ // # fall through to LoopMBB
+ MBB->addSuccessor(LoopMBB);
+
+ DestBase = MachineOperand::CreateReg(NextDestReg, false);
+ SrcBase = MachineOperand::CreateReg(NextSrcReg, false);
+ ImmLength &= 255;
+ if (EndMBB && !ImmLength)
+ // If the loop handled the whole CLC range, DoneMBB will be empty with
+ // CC live-through into EndMBB, so add it as live-in.
+ DoneMBB->addLiveIn(SystemZ::CC);
+ }
// LoopMBB:
// %ThisDestReg = phi [ %StartDestReg, StartMBB ],
@@ -7808,7 +7902,6 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
//
// The prefetch is used only for MVC. The JLH is used only for CLC.
MBB = LoopMBB;
-
BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg)
.addReg(StartDestReg).addMBB(StartMBB)
.addReg(NextDestReg).addMBB(NextMBB);
@@ -7840,11 +7933,10 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
// %NextCountReg = AGHI %ThisCountReg, -1
// CGHI %NextCountReg, 0
// JLH LoopMBB
- // # fall through to DoneMMB
+ // # fall through to DoneMBB
//
// The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
MBB = NextMBB;
-
BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg)
.addReg(ThisDestReg).addImm(256).addReg(0);
if (!HaveSingleBase)
@@ -7860,18 +7952,39 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
MBB->addSuccessor(LoopMBB);
MBB->addSuccessor(DoneMBB);
- DestBase = MachineOperand::CreateReg(NextDestReg, false);
- SrcBase = MachineOperand::CreateReg(NextSrcReg, false);
- Length &= 255;
- if (EndMBB && !Length)
- // If the loop handled the whole CLC range, DoneMBB will be empty with
- // CC live-through into EndMBB, so add it as live-in.
- DoneMBB->addLiveIn(SystemZ::CC);
MBB = DoneMBB;
+ if (LengthMO.isReg()) {
+ // DoneMBB:
+ // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
+ // # Use EXecute Relative Long for the remainder of the bytes. The target
+ // instruction of the EXRL will have a length field of 1 since 0 is an
+ // illegal value. The number of bytes processed becomes (%LenMinus1Reg &
+ // 0xff) + 1.
+ // # Fall through to AllDoneMBB.
+ Register RemSrcReg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
+ Register RemDestReg = HaveSingleBase ? RemSrcReg
+ : MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
+ BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg)
+ .addReg(StartDestReg).addMBB(StartMBB)
+ .addReg(NextDestReg).addMBB(LoopMBB);
+ if (!HaveSingleBase)
+ BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)
+ .addReg(StartSrcReg).addMBB(StartMBB)
+ .addReg(NextSrcReg).addMBB(LoopMBB);
+ MRI.constrainRegClass(LenMinus1Reg, &SystemZ::ADDR64BitRegClass);
+ BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))
+ .addImm(Opcode)
+ .addReg(LenMinus1Reg)
+ .addReg(RemDestReg).addImm(DestDisp)
+ .addReg(RemSrcReg).addImm(SrcDisp);
+ MBB->addSuccessor(AllDoneMBB);
+ MBB = AllDoneMBB;
+ }
}
+
// Handle any remaining bytes with straight-line code.
- while (Length > 0) {
- uint64_t ThisLength = std::min(Length, uint64_t(256));
+ while (ImmLength > 0) {
+ uint64_t ThisLength = std::min(ImmLength, uint64_t(256));
// The previous iteration might have created out-of-range displacements.
// Apply them using LAY if so.
if (!isUInt<12>(DestDisp)) {
@@ -7901,10 +8014,10 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
.setMemRefs(MI.memoperands());
DestDisp += ThisLength;
SrcDisp += ThisLength;
- Length -= ThisLength;
+ ImmLength -= ThisLength;
// If there's another CLC to go, branch to the end if a difference
// was found.
- if (EndMBB && Length > 0) {
+ if (EndMBB && ImmLength > 0) {
MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
BuildMI(MBB, DL, TII->get(SystemZ::BRC))
.addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
@@ -7949,7 +8062,7 @@ MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
// StartMBB:
- // # fall through to LoopMMB
+ // # fall through to LoopMBB
MBB->addSuccessor(LoopMBB);
// LoopMBB:
@@ -7958,7 +8071,7 @@ MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
// R0L = %CharReg
// %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
// JO LoopMBB
- // # fall through to DoneMMB
+ // # fall through to DoneMBB
//
// The load of R0L can be hoisted by post-RA LICM.
MBB = LoopMBB;
@@ -8395,6 +8508,7 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
return emitMemMemWrapper(MI, MBB, SystemZ::OC);
case SystemZ::XCSequence:
case SystemZ::XCLoop:
+ case SystemZ::XCLoopVarLen:
return emitMemMemWrapper(MI, MBB, SystemZ::XC);
case SystemZ::CLCSequence:
case SystemZ::CLCLoop:
generated by cgit v1.3 (git 2.53.0) at 2026-07-30 19:54:57 +0000