aboutsummaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm')
-rw-r--r--contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h7
-rw-r--r--contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAG.h13
-rw-r--r--contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h1
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/MachineFunction.cpp3
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp4
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp5
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h3
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp3
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp3
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp12
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp5
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp10
-rw-r--r--contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp1
-rw-r--r--contrib/llvm-project/llvm/lib/Object/ELF.cpp2
-rw-r--r--contrib/llvm-project/llvm/lib/Support/AArch64TargetParser.cpp2
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td12
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp2
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp34
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp8
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp163
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h9
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td11
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h6
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp16
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp12
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp7
-rw-r--r--contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp1
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp63
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp2
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp2
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h4
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp4
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp13
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp1
-rw-r--r--contrib/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td4
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp8
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp54
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp23
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp71
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp30
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h25
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp23
-rw-r--r--contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.h8
-rw-r--r--contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp24
-rw-r--r--contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp6
-rw-r--r--contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp17
-rw-r--r--contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp37
-rw-r--r--contrib/llvm-project/llvm/lib/Transforms/Vectorize/VectorCombine.cpp8
-rw-r--r--contrib/llvm-project/llvm/tools/llvm-objdump/ELFDump.cpp8
49 files changed, 525 insertions, 265 deletions
diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h
index 775698a66ada..6de8ac4273f7 100644
--- a/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h
+++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h
@@ -212,6 +212,7 @@ protected:
const TargetRegisterInfo &TRI;
const TargetLibraryInfo *LibInfo;
bool SkipTargetIndependentISel;
+ bool UseInstrRefDebugInfo = false;
/// The position of the last instruction for materializing constants
/// for use in the current block. It resets to EmitStartPt when it makes sense
@@ -318,6 +319,12 @@ public:
/// Reset InsertPt to the given old insert position.
void leaveLocalValueArea(SavePoint Old);
+ /// Signal whether instruction referencing variable locations are desired for
+ /// this function's debug-info.
+ void useInstrRefDebugInfo(bool Flag) {
+ UseInstrRefDebugInfo = Flag;
+ }
+
protected:
explicit FastISel(FunctionLoweringInfo &FuncInfo,
const TargetLibraryInfo *LibInfo,
diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAG.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAG.h
index e31719bcff0b..4f348c9feaa5 100644
--- a/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -278,6 +278,9 @@ class SelectionDAG {
uint16_t NextPersistentId = 0;
+ /// Are instruction referencing variable locations desired for this function?
+ bool UseInstrRefDebugInfo = false;
+
public:
/// Clients of various APIs that cause global effects on
/// the DAG can optionally implement this interface. This allows the clients
@@ -1702,6 +1705,16 @@ public:
/// function mirrors \c llvm::salvageDebugInfo.
void salvageDebugInfo(SDNode &N);
+ /// Signal whether instruction referencing variable locations are desired for
+ /// this function's debug-info.
+ void useInstrRefDebugInfo(bool Flag) {
+ UseInstrRefDebugInfo = Flag;
+ }
+
+ bool getUseInstrRefDebugInfo() const {
+ return UseInstrRefDebugInfo;
+ }
+
void dump() const;
/// In most cases this function returns the ABI alignment for a given type,
diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 9cea197724cc..fc3fdf3e4583 100644
--- a/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -53,6 +53,7 @@ public:
const TargetLowering *TLI;
bool FastISelFailed;
SmallPtrSet<const Instruction *, 4> ElidedArgCopyInstrs;
+ bool UseInstrRefDebugInfo = false;
/// Current optimization remark emitter.
/// Used to report things like combines and FastISel failures.
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineFunction.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineFunction.cpp
index fd5ea5cad072..02f58ca5eef0 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineFunction.cpp
@@ -1181,9 +1181,6 @@ void MachineFunction::finalizeDebugInstrRefs() {
MI.getOperand(1).ChangeToRegister(0, false);
};
- if (!useDebugInstrRef())
- return;
-
for (auto &MBB : *this) {
for (auto &MI : MBB) {
if (!MI.isDebugRef() || !MI.getOperand(0).isReg())
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index d8ef79fe9a7b..87a1ebe4c1db 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1265,7 +1265,7 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
// If using instruction referencing, mutate this into a DBG_INSTR_REF,
// to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
// the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
- if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
+ if (UseInstrRefDebugInfo && Op->isReg()) {
Builder->setDesc(TII.get(TargetOpcode::DBG_INSTR_REF));
Builder->getOperand(1).ChangeToImmediate(0);
auto *NewExpr =
@@ -1324,7 +1324,7 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
// If using instruction referencing, mutate this into a DBG_INSTR_REF,
// to be later patched up by finalizeDebugInstrRefs.
- if (FuncInfo.MF->useDebugInstrRef()) {
+ if (UseInstrRefDebugInfo) {
Builder->setDesc(TII.get(TargetOpcode::DBG_INSTR_REF));
Builder->getOperand(1).ChangeToImmediate(0);
}
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 331e0325aea3..e3e05c868102 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -1341,11 +1341,12 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
/// InstrEmitter - Construct an InstrEmitter and set it to start inserting
/// at the given position in the given block.
InstrEmitter::InstrEmitter(const TargetMachine &TM, MachineBasicBlock *mbb,
- MachineBasicBlock::iterator insertpos)
+ MachineBasicBlock::iterator insertpos,
+ bool UseInstrRefDebugInfo)
: MF(mbb->getParent()), MRI(&MF->getRegInfo()),
TII(MF->getSubtarget().getInstrInfo()),
TRI(MF->getSubtarget().getRegisterInfo()),
TLI(MF->getSubtarget().getTargetLowering()), MBB(mbb),
InsertPos(insertpos) {
- EmitDebugInstrRefs = MF->useDebugInstrRef();
+ EmitDebugInstrRefs = UseInstrRefDebugInfo;
}
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
index ac8a70156522..ced8f064b9be 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -154,7 +154,8 @@ public:
/// InstrEmitter - Construct an InstrEmitter and set it to start inserting
/// at the given position in the given block.
InstrEmitter(const TargetMachine &TM, MachineBasicBlock *mbb,
- MachineBasicBlock::iterator insertpos);
+ MachineBasicBlock::iterator insertpos,
+ bool UseInstrRefDebugInfo);
private:
void EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
index 1b89864116cb..1a6be0cc2091 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
@@ -758,7 +758,8 @@ void ScheduleDAGLinearize::Schedule() {
MachineBasicBlock*
ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
- InstrEmitter Emitter(DAG->getTarget(), BB, InsertPos);
+ InstrEmitter Emitter(DAG->getTarget(), BB, InsertPos,
+ DAG->getUseInstrRefDebugInfo());
DenseMap<SDValue, Register> VRBaseMap;
LLVM_DEBUG({ dbgs() << "\n*** Final schedule ***\n"; });
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 55f6f288f3e3..92897aca7f6b 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -843,7 +843,8 @@ EmitPhysRegCopy(SUnit *SU, DenseMap<SUnit*, Register> &VRBaseMap,
/// not necessarily refer to returned BB. The emitter may split blocks.
MachineBasicBlock *ScheduleDAGSDNodes::
EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
- InstrEmitter Emitter(DAG->getTarget(), BB, InsertPos);
+ InstrEmitter Emitter(DAG->getTarget(), BB, InsertPos,
+ DAG->getUseInstrRefDebugInfo());
DenseMap<SDValue, Register> VRBaseMap;
DenseMap<SUnit*, Register> CopyVRBaseMap;
SmallVector<std::pair<unsigned, MachineInstr*>, 32> Orders;
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index d5998d166d25..40d861702e86 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5494,8 +5494,18 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
// Build vector (integer) scalar operands may need implicit
// truncation - do this before constant folding.
- if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
+ if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {
+ // Don't create illegally-typed nodes unless they're constants or undef
+ // - if we fail to constant fold we can't guarantee the (dead) nodes
+ // we're creating will be cleaned up before being visited for
+ // legalization.
+ if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
+ !isa<ConstantSDNode>(ScalarOp) &&
+ TLI->getTypeAction(*getContext(), InSVT) !=
+ TargetLowering::TypeLegal)
+ return SDValue();
ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
+ }
ScalarOps.push_back(ScalarOp);
}
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 01230a36e744..c61716ba1676 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -926,7 +926,10 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
CallConv.getValue(), RegVTs[Value])
: RegVTs[Value];
- if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
+ // We need to zero extend constants that are liveout to match assumptions
+ // in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
+ if (ExtendKind == ISD::ANY_EXTEND &&
+ (TLI.isZExtFree(Val, RegisterVT) || isa<ConstantSDNode>(Val)))
ExtendKind = ISD::ZERO_EXTEND;
getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 3c786904620a..b83a60129c78 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -425,6 +425,11 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
const Function &Fn = mf.getFunction();
MF = &mf;
+ // Decide what flavour of variable location debug-info will be used, before
+ // we change the optimisation level.
+ UseInstrRefDebugInfo = mf.useDebugInstrRef();
+ CurDAG->useInstrRefDebugInfo(UseInstrRefDebugInfo);
+
// Reset the target options before resetting the optimization
// level below.
// FIXME: This is a horrible hack and should be processed via
@@ -654,7 +659,8 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
// For debug-info, in instruction referencing mode, we need to perform some
// post-isel maintenence.
- MF->finalizeDebugInstrRefs();
+ if (UseInstrRefDebugInfo)
+ MF->finalizeDebugInstrRefs();
// Determine if there are any calls in this machine function.
MachineFrameInfo &MFI = MF->getFrameInfo();
@@ -1380,6 +1386,8 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
if (TM.Options.EnableFastISel) {
LLVM_DEBUG(dbgs() << "Enabling fast-isel\n");
FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
+ if (FastIS)
+ FastIS->useInstrRefDebugInfo(UseInstrRefDebugInfo);
}
ReversePostOrderTraversal<const Function*> RPOT(&Fn);
diff --git a/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp b/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
index 883735fcc293..6fd2f7e7a718 100644
--- a/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1336,6 +1336,7 @@ bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
// can update it.
return true;
case ELF::STB_GLOBAL:
+ case ELF::STB_GNU_UNIQUE:
// Global ELF symbols can be preempted by the dynamic linker. The relocation
// has to point to the symbol for a reason analogous to the STB_WEAK case.
return true;
diff --git a/contrib/llvm-project/llvm/lib/Object/ELF.cpp b/contrib/llvm-project/llvm/lib/Object/ELF.cpp
index 6e56da1a31f3..56a426211755 100644
--- a/contrib/llvm-project/llvm/lib/Object/ELF.cpp
+++ b/contrib/llvm-project/llvm/lib/Object/ELF.cpp
@@ -561,11 +561,9 @@ Expected<typename ELFT::DynRange> ELFFile<ELFT>::dynamicEntries() const {
}
if (Dyn.empty())
- // TODO: this error is untested.
return createError("invalid empty dynamic section");
if (Dyn.back().d_tag != ELF::DT_NULL)
- // TODO: this error is untested.
return createError("dynamic sections must be DT_NULL terminated");
return Dyn;
diff --git a/contrib/llvm-project/llvm/lib/Support/AArch64TargetParser.cpp b/contrib/llvm-project/llvm/lib/Support/AArch64TargetParser.cpp
index cdf7c8ade9aa..bb19e2714be1 100644
--- a/contrib/llvm-project/llvm/lib/Support/AArch64TargetParser.cpp
+++ b/contrib/llvm-project/llvm/lib/Support/AArch64TargetParser.cpp
@@ -120,6 +120,8 @@ bool AArch64::getExtensionFeatures(uint64_t Extensions,
Features.push_back("+mops");
if (Extensions & AArch64::AEK_PERFMON)
Features.push_back("+perfmon");
+ if (Extensions & AArch64::AEK_SSBS)
+ Features.push_back("+ssbs");
return true;
}
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td
index 9a04b28a8b8f..70c7b7b3f5dc 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td
@@ -464,6 +464,11 @@ def FeatureEL3 : SubtargetFeature<"el3", "HasEL3", "true",
def FeatureFixCortexA53_835769 : SubtargetFeature<"fix-cortex-a53-835769",
"FixCortexA53_835769", "true", "Mitigate Cortex-A53 Erratum 835769">;
+def FeatureNoBTIAtReturnTwice : SubtargetFeature<"no-bti-at-return-twice",
+ "NoBTIAtReturnTwice", "true",
+ "Don't place a BTI instruction "
+ "after a return-twice">;
+
//===----------------------------------------------------------------------===//
// Architectures.
//
@@ -953,7 +958,7 @@ def ProcessorFeatures {
FeatureRCPC, FeatureSSBS];
list<SubtargetFeature> A77 = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
FeatureNEON, FeatureFullFP16, FeatureDotProd,
- FeatureRCPC];
+ FeatureRCPC, FeatureSSBS];
list<SubtargetFeature> A78 = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
FeatureNEON, FeatureFullFP16, FeatureDotProd,
FeatureRCPC, FeaturePerfMon, FeatureSPE,
@@ -971,11 +976,12 @@ def ProcessorFeatures {
FeatureSB, FeatureSpecRestrict];
list<SubtargetFeature> X1 = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
FeatureNEON, FeatureRCPC, FeaturePerfMon,
- FeatureSPE, FeatureFullFP16, FeatureDotProd];
+ FeatureSPE, FeatureFullFP16, FeatureDotProd,
+ FeatureSSBS];
list<SubtargetFeature> X1C = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
FeatureNEON, FeatureRCPC, FeaturePerfMon,
FeatureSPE, FeatureFullFP16, FeatureDotProd,
- FeaturePAuth];
+ FeaturePAuth, FeatureSSBS];
list<SubtargetFeature> X2 = [HasV9_0aOps, FeatureNEON, FeaturePerfMon,
FeatureMatMulInt8, FeatureBF16, FeatureAM,
FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp
index ac243347b24d..b31b709c0c0a 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp
@@ -559,7 +559,7 @@ bool AArch64CollectLOH::runOnMachineFunction(MachineFunction &MF) {
// Walk the basic block backwards and update the per register state machine
// in the process.
for (const MachineInstr &MI :
- instructionsWithoutDebug(MBB.rbegin(), MBB.rend())) {
+ instructionsWithoutDebug(MBB.instr_rbegin(), MBB.instr_rend())) {
unsigned Opcode = MI.getOpcode();
switch (Opcode) {
case AArch64::ADDXri:
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index b0f739cc26e6..910f8cdede75 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -86,6 +86,7 @@ private:
unsigned N);
bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
+ bool expandCALL_BTI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
};
@@ -759,6 +760,37 @@ bool AArch64ExpandPseudo::expandCALL_RVMARKER(
return true;
}
+bool AArch64ExpandPseudo::expandCALL_BTI(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI) {
+ // Expand CALL_BTI pseudo to:
+ // - a branch to the call target
+ // - a BTI instruction
+ // Mark the sequence as a bundle, to avoid passes moving other code in
+ // between.
+
+ MachineInstr &MI = *MBBI;
+ MachineOperand &CallTarget = MI.getOperand(0);
+ assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
+ "invalid operand for regular call");
+ unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
+ MachineInstr *Call =
+ BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr();
+ Call->addOperand(CallTarget);
+
+ MachineInstr *BTI =
+ BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::HINT))
+ // BTI J so that setjmp can to BR to this.
+ .addImm(36)
+ .getInstr();
+
+ if (MI.shouldUpdateCallSiteInfo())
+ MBB.getParent()->moveCallSiteInfo(&MI, Call);
+
+ MI.eraseFromParent();
+ finalizeBundle(MBB, Call->getIterator(), std::next(BTI->getIterator()));
+ return true;
+}
+
bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
Register CtxReg = MBBI->getOperand(0).getReg();
@@ -1238,6 +1270,8 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2);
case AArch64::BLR_RVMARKER:
return expandCALL_RVMARKER(MBB, MBBI);
+ case AArch64::BLR_BTI:
+ return expandCALL_BTI(MBB, MBBI);
case AArch64::StoreSwiftAsyncContext:
return expandStoreSwiftAsyncContext(MBB, MBBI);
}
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index c67fa62c7a92..dc5e6807945d 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -14,6 +14,7 @@
#include "AArch64.h"
#include "AArch64CallingConvention.h"
+#include "AArch64MachineFunctionInfo.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
@@ -3127,6 +3128,13 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
if (!Callee && !Symbol)
return false;
+ // Allow SelectionDAG isel to handle calls to functions like setjmp that need
+ // a bti instruction following the call.
+ if (CLI.CB && CLI.CB->hasFnAttr(Attribute::ReturnsTwice) &&
+ !Subtarget->noBTIAtReturnTwice() &&
+ MF->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
+ return false;
+
// Allow SelectionDAG isel to handle tail calls.
if (IsTailCall)
return false;
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ac5e51e47ddf..4d1cb0720a5a 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2278,6 +2278,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
MAKE_CASE(AArch64ISD::MOPS_MEMSET_TAGGING)
MAKE_CASE(AArch64ISD::MOPS_MEMCOPY)
MAKE_CASE(AArch64ISD::MOPS_MEMMOVE)
+ MAKE_CASE(AArch64ISD::CALL_BTI)
}
#undef MAKE_CASE
return nullptr;
@@ -5843,14 +5844,62 @@ static bool mayTailCallThisCC(CallingConv::ID CC) {
}
}
+static void analyzeCallOperands(const AArch64TargetLowering &TLI,
+ const AArch64Subtarget *Subtarget,
+ const TargetLowering::CallLoweringInfo &CLI,
+ CCState &CCInfo) {
+ const SelectionDAG &DAG = CLI.DAG;
+ CallingConv::ID CalleeCC = CLI.CallConv;
+ bool IsVarArg = CLI.IsVarArg;
+ const SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
+ bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CalleeCC);
+
+ unsigned NumArgs = Outs.size();
+ for (unsigned i = 0; i != NumArgs; ++i) {
+ MVT ArgVT = Outs[i].VT;
+ ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
+
+ bool UseVarArgCC = false;
+ if (IsVarArg) {
+ // On Windows, the fixed arguments in a vararg call are passed in GPRs
+ // too, so use the vararg CC to force them to integer registers.
+ if (IsCalleeWin64) {
+ UseVarArgCC = true;
+ } else {
+ UseVarArgCC = !Outs[i].IsFixed;
+ }
+ } else {
+ // Get type of the original argument.
+ EVT ActualVT =
+ TLI.getValueType(DAG.getDataLayout(), CLI.Args[Outs[i].OrigArgIndex].Ty,
+ /*AllowUnknown*/ true);
+ MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ArgVT;
+ // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
+ if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
+ ArgVT = MVT::i8;
+ else if (ActualMVT == MVT::i16)
+ ArgVT = MVT::i16;
+ }
+
+ CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CalleeCC, UseVarArgCC);
+ bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
+ assert(!Res && "Call operand has unhandled type");
+ (void)Res;
+ }
+}
+
bool AArch64TargetLowering::isEligibleForTailCallOptimization(
- SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
- const SmallVectorImpl<ISD::OutputArg> &Outs,
- const SmallVectorImpl<SDValue> &OutVals,
- const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
+ const CallLoweringInfo &CLI) const {
+ CallingConv::ID CalleeCC = CLI.CallConv;
if (!mayTailCallThisCC(CalleeCC))
return false;
+ SDValue Callee = CLI.Callee;
+ bool IsVarArg = CLI.IsVarArg;
+ const SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
+ const SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
+ const SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
+ const SelectionDAG &DAG = CLI.DAG;
MachineFunction &MF = DAG.getMachineFunction();
const Function &CallerF = MF.getFunction();
CallingConv::ID CallerCC = CallerF.getCallingConv();
@@ -5915,30 +5964,14 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization(
// I want anyone implementing a new calling convention to think long and hard
// about this assert.
- assert((!isVarArg || CalleeCC == CallingConv::C) &&
+ assert((!IsVarArg || CalleeCC == CallingConv::C) &&
"Unexpected variadic calling convention");
LLVMContext &C = *DAG.getContext();
- if (isVarArg && !Outs.empty()) {
- // At least two cases here: if caller is fastcc then we can't have any
- // memory arguments (we'd be expected to clean up the stack afterwards). If
- // caller is C then we could potentially use its argument area.
-
- // FIXME: for now we take the most conservative of these in both cases:
- // disallow all variadic memory operands.
- SmallVector<CCValAssign, 16> ArgLocs;
- CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
-
- CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true));
- for (const CCValAssign &ArgLoc : ArgLocs)
- if (!ArgLoc.isRegLoc())
- return false;
- }
-
// Check that the call results are passed in the same way.
if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
- CCAssignFnForCall(CalleeCC, isVarArg),
- CCAssignFnForCall(CallerCC, isVarArg)))
+ CCAssignFnForCall(CalleeCC, IsVarArg),
+ CCAssignFnForCall(CallerCC, IsVarArg)))
return false;
// The callee has to preserve all registers the caller needs to preserve.
const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
@@ -5958,9 +5991,22 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization(
return true;
SmallVector<CCValAssign, 16> ArgLocs;
- CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
+ CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, C);
- CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
+ analyzeCallOperands(*this, Subtarget, CLI, CCInfo);
+
+ if (IsVarArg && !(CLI.CB && CLI.CB->isMustTailCall())) {
+ // When we are musttail, additional checks have been done and we can safely ignore this check
+ // At least two cases here: if caller is fastcc then we can't have any
+ // memory arguments (we'd be expected to clean up the stack afterwards). If
+ // caller is C then we could potentially use its argument area.
+
+ // FIXME: for now we take the most conservative of these in both cases:
+ // disallow all variadic memory operands.
+ for (const CCValAssign &ArgLoc : ArgLocs)
+ if (!ArgLoc.isRegLoc())
+ return false;
+ }
const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
@@ -6051,7 +6097,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
SDValue Chain = CLI.Chain;
SDValue Callee = CLI.Callee;
bool &IsTailCall = CLI.IsTailCall;
- CallingConv::ID CallConv = CLI.CallConv;
+ CallingConv::ID &CallConv = CLI.CallConv;
bool IsVarArg = CLI.IsVarArg;
MachineFunction &MF = DAG.getMachineFunction();
@@ -6061,7 +6107,12 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
bool IsSibCall = false;
- bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CallConv);
+ bool GuardWithBTI = false;
+
+ if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) &&
+ !Subtarget->noBTIAtReturnTwice()) {
+ GuardWithBTI = FuncInfo->branchTargetEnforcement();
+ }
// Check callee args/returns for SVE registers and set calling convention
// accordingly.
@@ -6079,8 +6130,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
if (IsTailCall) {
// Check if it's really possible to do a tail call.
- IsTailCall = isEligibleForTailCallOptimization(
- Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
+ IsTailCall = isEligibleForTailCallOptimization(CLI);
// A sibling call is one where we're under the usual C ABI and not planning
// to change that but can still do a tail call:
@@ -6101,56 +6151,17 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
if (IsVarArg) {
- // Handle fixed and variable vector arguments differently.
- // Variable vector arguments always go into memory.
unsigned NumArgs = Outs.size();
for (unsigned i = 0; i != NumArgs; ++i) {
- MVT ArgVT = Outs[i].VT;
- if (!Outs[i].IsFixed && ArgVT.isScalableVector())
+ if (!Outs[i].IsFixed && Outs[i].VT.isScalableVector())
report_fatal_error("Passing SVE types to variadic functions is "
"currently not supported");
-
- ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
- bool UseVarArgCC = !Outs[i].IsFixed;
- // On Windows, the fixed arguments in a vararg call are passed in GPRs
- // too, so use the vararg CC to force them to integer registers.
- if (IsCalleeWin64)
- UseVarArgCC = true;
- CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, UseVarArgCC);
- bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
- assert(!Res && "Call operand has unhandled type");
- (void)Res;
- }
- } else {
- // At this point, Outs[].VT may already be promoted to i32. To correctly
- // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
- // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
- // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
- // we use a special version of AnalyzeCallOperands to pass in ValVT and
- // LocVT.
- unsigned NumArgs = Outs.size();
- for (unsigned i = 0; i != NumArgs; ++i) {
- MVT ValVT = Outs[i].VT;
- // Get type of the original argument.
- EVT ActualVT = getValueType(DAG.getDataLayout(),
- CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
- /*AllowUnknown*/ true);
- MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
- ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
- // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
- if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
- ValVT = MVT::i8;
- else if (ActualMVT == MVT::i16)
- ValVT = MVT::i16;
-
- CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
- bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo);
- assert(!Res && "Call operand has unhandled type");
- (void)Res;
}
}
+ analyzeCallOperands(*this, Subtarget, CLI, CCInfo);
+
// Get a count of how many bytes are to be pushed on the stack.
unsigned NumBytes = CCInfo.getNextStackOffset();
@@ -6536,7 +6547,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
Function *ARCFn = *objcarc::getAttachedARCFunction(CLI.CB);
auto GA = DAG.getTargetGlobalAddress(ARCFn, DL, PtrVT);
Ops.insert(Ops.begin() + 1, GA);
- }
+ } else if (GuardWithBTI)
+ CallOpc = AArch64ISD::CALL_BTI;
// Returns a chain and a flag for retval copy to use.
Chain = DAG.getNode(CallOpc, DL, NodeTys, Ops);
@@ -17111,13 +17123,14 @@ static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
// Check whether folding this offset is legal. It must not go out of bounds of
// the referenced object to avoid violating the code model, and must be
- // smaller than 2^21 because this is the largest offset expressible in all
- // object formats.
+ // smaller than 2^20 because this is the largest offset expressible in all
+ // object formats. (The IMAGE_REL_ARM64_PAGEBASE_REL21 relocation in COFF
+ // stores an immediate signed 21 bit offset.)
//
// This check also prevents us from folding negative offsets, which will end
// up being treated in the same way as large positive ones. They could also
// cause code model violations, and aren't really common enough to matter.
- if (Offset >= (1 << 21))
+ if (Offset >= (1 << 20))
return SDValue();
const GlobalValue *GV = GN->getGlobal();
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 2138c0ffe70a..80b7e84872cd 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -55,6 +55,8 @@ enum NodeType : unsigned {
// x29, x29` marker instruction.
CALL_RVMARKER,
+ CALL_BTI, // Function call followed by a BTI instruction.
+
// Produces the full sequence of instructions for getting the thread pointer
// offset of a variable into X0, using the TLSDesc model.
TLSDESC_CALLSEQ,
@@ -898,11 +900,8 @@ private:
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
- bool isEligibleForTailCallOptimization(
- SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
- const SmallVectorImpl<ISD::OutputArg> &Outs,
- const SmallVectorImpl<SDValue> &OutVals,
- const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
+ bool
+ isEligibleForTailCallOptimization(const CallLoweringInfo &CLI) const;
/// Finds the incoming stack arguments which overlap the given fixed stack
/// object and incorporates their load into the current chain. This prevents
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 1316161f05f1..2680b5ac094e 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -473,6 +473,11 @@ def AArch64call : SDNode<"AArch64ISD::CALL",
[SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
SDNPVariadic]>;
+def AArch64call_bti : SDNode<"AArch64ISD::CALL_BTI",
+ SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
+ [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+ SDNPVariadic]>;
+
def AArch64call_rvmarker: SDNode<"AArch64ISD::CALL_RVMARKER",
SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
[SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
@@ -2320,6 +2325,8 @@ let isCall = 1, Defs = [LR], Uses = [SP] in {
PseudoInstExpansion<(BLR GPR64:$Rn)>;
def BLR_RVMARKER : Pseudo<(outs), (ins variable_ops), []>,
Sched<[WriteBrReg]>;
+ def BLR_BTI : Pseudo<(outs), (ins variable_ops), []>,
+ Sched<[WriteBrReg]>;
} // isCall
def : Pat<(AArch64call GPR64:$Rn),
@@ -2333,6 +2340,10 @@ def : Pat<(AArch64call_rvmarker (i64 tglobaladdr:$rvfunc), GPR64:$Rn),
(BLR_RVMARKER tglobaladdr:$rvfunc, GPR64:$Rn)>,
Requires<[NoSLSBLRMitigation]>;
+def : Pat<(AArch64call_bti GPR64:$Rn),
+ (BLR_BTI GPR64:$Rn)>,
+ Requires<[NoSLSBLRMitigation]>;
+
let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
def BR : BranchReg<0b0000, "br", [(brind GPR64:$Rn)]>;
} // isBranch, isTerminator, isBarrier, isIndirectBranch
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 7b2bbad30f85..061db926ee2b 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -243,6 +243,10 @@ protected:
// Enable 64-bit vectorization in SLP.
unsigned MinVectorRegisterBitWidth = 64;
+ // Do not place a BTI instruction after a call to a return twice function like
+ // setjmp.
+ bool NoBTIAtReturnTwice = false;
+
bool OutlineAtomics = false;
bool PredictableSelectIsExpensive = false;
bool BalanceFPOps = false;
@@ -588,6 +592,8 @@ public:
bool fixCortexA53_835769() const { return FixCortexA53_835769; }
+ bool noBTIAtReturnTwice() const { return NoBTIAtReturnTwice; }
+
bool addrSinkUsingGEPs() const override {
// Keeping GEPs inbounds is important for exploiting AArch64
// addressing-modes in ILP32 mode.
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 33ed7ae9780e..8f71eb3095cf 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -6515,21 +6515,13 @@ bool AArch64AsmParser::parseDirectiveCFIBKeyFrame() {
/// parseDirectiveVariantPCS
/// ::= .variant_pcs symbolname
bool AArch64AsmParser::parseDirectiveVariantPCS(SMLoc L) {
- const AsmToken &Tok = getTok();
- if (Tok.isNot(AsmToken::Identifier))
+ StringRef Name;
+ if (getParser().parseIdentifier(Name))
return TokError("expected symbol name");
-
- StringRef SymbolName = Tok.getIdentifier();
-
- MCSymbol *Sym = getContext().lookupSymbol(SymbolName);
- if (!Sym)
- return TokError("unknown symbol");
-
- Lex(); // Eat the symbol
-
if (parseEOL())
return true;
- getTargetStreamer().emitDirectiveVariantPCS(Sym);
+ getTargetStreamer().emitDirectiveVariantPCS(
+ getContext().getOrCreateSymbol(Name));
return false;
}
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 097b93e4fcca..bb6c7938791e 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -1127,14 +1127,22 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
// Create a temporarily-floating call instruction so we can add the implicit
// uses of arg registers.
- unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
+ const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+ unsigned Opc = 0;
+ // A call to a returns twice function like setjmp must be followed by a bti
+ // instruction.
+ if (Info.CB && Info.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) &&
+ !Subtarget.noBTIAtReturnTwice() &&
+ MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
+ Opc = AArch64::BLR_BTI;
+ else
+ Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
MIB.add(Info.Callee);
// Tell the call which registers are clobbered.
const uint32_t *Mask;
- const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const auto *TRI = Subtarget.getRegisterInfo();
AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index d3f4130d2ba1..3730030aa140 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -162,13 +162,14 @@ static bool matchFoldGlobalOffset(MachineInstr &MI, MachineRegisterInfo &MRI,
// Check whether folding this offset is legal. It must not go out of bounds of
// the referenced object to avoid violating the code model, and must be
- // smaller than 2^21 because this is the largest offset expressible in all
- // object formats.
+ // smaller than 2^20 because this is the largest offset expressible in all
+ // object formats. (The IMAGE_REL_ARM64_PAGEBASE_REL21 relocation in COFF
+ // stores an immediate signed 21 bit offset.)
//
// This check also prevents us from folding negative offsets, which will end
// up being treated in the same way as large positive ones. They could also
// cause code model violations, and aren't really common enough to matter.
- if (NewOffset >= (1 << 21))
+ if (NewOffset >= (1 << 20))
return false;
Type *T = GV->getValueType();
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 78c0e90b1384..46edb12959d2 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -254,6 +254,7 @@ void AArch64TargetELFStreamer::emitInst(uint32_t Inst) {
}
void AArch64TargetELFStreamer::emitDirectiveVariantPCS(MCSymbol *Symbol) {
+ getStreamer().getAssembler().registerSymbol(*Symbol);
cast<MCSymbolELF>(Symbol)->setOther(ELF::STO_AARCH64_VARIANT_PCS);
}
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 715cff72dcab..7113fe33b5d7 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -341,31 +341,11 @@ public:
bool isU10Imm() const { return Kind == Immediate && isUInt<10>(getImm()); }
bool isU12Imm() const { return Kind == Immediate && isUInt<12>(getImm()); }
- bool isU16Imm() const {
- switch (Kind) {
- case Expression:
- return true;
- case Immediate:
- case ContextImmediate:
- return isUInt<16>(getImmU16Context());
- default:
- return false;
- }
- }
- bool isS16Imm() const {
- switch (Kind) {
- case Expression:
- return true;
- case Immediate:
- case ContextImmediate:
- return isInt<16>(getImmS16Context());
- default:
- return false;
- }
- }
- bool isS16ImmX4() const { return Kind == Expression ||
- (Kind == Immediate && isInt<16>(getImm()) &&
- (getImm() & 3) == 0); }
+ bool isU16Imm() const { return isExtImm<16>(/*Signed*/ false, 1); }
+ bool isS16Imm() const { return isExtImm<16>(/*Signed*/ true, 1); }
+ bool isS16ImmX4() const { return isExtImm<16>(/*Signed*/ true, 4); }
+ bool isS16ImmX16() const { return isExtImm<16>(/*Signed*/ true, 16); }
+ bool isS17Imm() const { return isExtImm<17>(/*Signed*/ true, 1); }
bool isHashImmX8() const {
// The Hash Imm form is used for instructions that check or store a hash.
@@ -375,9 +355,6 @@ public:
(getImm() & 7) == 0);
}
- bool isS16ImmX16() const { return Kind == Expression ||
- (Kind == Immediate && isInt<16>(getImm()) &&
- (getImm() & 15) == 0); }
bool isS34ImmX16() const {
return Kind == Expression ||
(Kind == Immediate && isInt<34>(getImm()) && (getImm() & 15) == 0);
@@ -388,17 +365,6 @@ public:
return Kind == Expression || (Kind == Immediate && isInt<34>(getImm()));
}
- bool isS17Imm() const {
- switch (Kind) {
- case Expression:
- return true;
- case Immediate:
- case ContextImmediate:
- return isInt<17>(getImmS16Context());
- default:
- return false;
- }
- }
bool isTLSReg() const { return Kind == TLSRegister; }
bool isDirectBr() const {
if (Kind == Expression)
@@ -712,6 +678,25 @@ public:
return CreateExpr(Val, S, E, IsPPC64);
}
+
+private:
+ template <unsigned Width>
+ bool isExtImm(bool Signed, unsigned Multiple) const {
+ switch (Kind) {
+ default:
+ return false;
+ case Expression:
+ return true;
+ case Immediate:
+ case ContextImmediate:
+ if (Signed)
+ return isInt<Width>(getImmS16Context()) &&
+ (getImmS16Context() & (Multiple - 1)) == 0;
+ else
+ return isUInt<Width>(getImmU16Context()) &&
+ (getImmU16Context() & (Multiple - 1)) == 0;
+ }
+ }
};
} // end anonymous namespace.
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 9df94edc8cdf..2e678ffd58c2 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -44,6 +44,7 @@ static uint64_t adjustFixupValue(unsigned Kind, uint64_t Value) {
case PPC::fixup_ppc_half16:
return Value & 0xffff;
case PPC::fixup_ppc_half16ds:
+ case PPC::fixup_ppc_half16dq:
return Value & 0xfffc;
case PPC::fixup_ppc_pcrel34:
case PPC::fixup_ppc_imm34:
@@ -60,6 +61,7 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
case FK_Data_2:
case PPC::fixup_ppc_half16:
case PPC::fixup_ppc_half16ds:
+ case PPC::fixup_ppc_half16dq:
return 2;
case FK_Data_4:
case PPC::fixup_ppc_brcond14:
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index 94ef7b45434f..1e58039582c2 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -125,6 +125,7 @@ unsigned PPCELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
}
break;
case PPC::fixup_ppc_half16ds:
+ case PPC::fixup_ppc_half16dq:
Target.print(errs());
errs() << '\n';
report_fatal_error("Invalid PC-relative half16ds relocation");
@@ -349,6 +350,7 @@ unsigned PPCELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
}
break;
case PPC::fixup_ppc_half16ds:
+ case PPC::fixup_ppc_half16dq:
switch (Modifier) {
default: llvm_unreachable("Unsupported Modifier");
case MCSymbolRefExpr::VK_None:
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
index 73292f7b7938..df0c666f5b11 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
@@ -51,6 +51,10 @@ enum Fixups {
/// register number.
fixup_ppc_nofixup,
+ /// A 16-bit fixup corresponding to lo16(_foo) with implied 3 zero bits for
+ /// instrs like 'lxv'. Produces the same relocation as fixup_ppc_half16ds.
+ fixup_ppc_half16dq,
+
// Marker
LastTargetFixupKind,
NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index 4dfa7d5e600c..d76795fcebc4 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -198,8 +198,8 @@ unsigned PPCMCCodeEmitter::getMemRIX16Encoding(const MCInst &MI, unsigned OpNo,
}
// Otherwise add a fixup for the displacement field.
- Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
- (MCFixupKind)PPC::fixup_ppc_half16ds));
+ Fixups.push_back(MCFixup::create(IsLittleEndian ? 0 : 2, MO.getExpr(),
+ (MCFixupKind)PPC::fixup_ppc_half16dq));
return RegBits;
}
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
index abff44449131..6cd04ee018fd 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
@@ -110,9 +110,18 @@ PPCMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
if (Value.isAbsolute()) {
int64_t Result = evaluateAsInt64(Value.getConstant());
- if ((Fixup == nullptr || (unsigned)Fixup->getKind() != PPC::fixup_ppc_half16) &&
- (Result >= 0x8000))
+ bool IsHalf16 = Fixup && Fixup->getTargetKind() == PPC::fixup_ppc_half16;
+ bool IsHalf16DS =
+ Fixup && Fixup->getTargetKind() == PPC::fixup_ppc_half16ds;
+ bool IsHalf16DQ =
+ Fixup && Fixup->getTargetKind() == PPC::fixup_ppc_half16dq;
+ bool IsHalf = IsHalf16 || IsHalf16DS || IsHalf16DQ;
+
+ if (!IsHalf && Result >= 0x8000)
return false;
+ if ((IsHalf16DS && (Result & 0x3)) || (IsHalf16DQ && (Result & 0xf)))
+ return false;
+
Res = MCValue::get(Result);
} else {
if (!Layout)
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index de9c5712a877..d9d37638c8a7 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -1305,6 +1305,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
}
+ setLibcallName(RTLIB::MULO_I128, nullptr);
if (!isPPC64) {
// These libcalls are not available in 32-bit.
setLibcallName(RTLIB::SHL_I128, nullptr);
diff --git a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index c26b4f6ceb7d..53e73e33b003 100644
--- a/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -1016,7 +1016,7 @@ def dispRI : Operand<iPTR> {
}
def PPCDispRIXOperand : AsmOperandClass {
let Name = "DispRIX"; let PredicateMethod = "isS16ImmX4";
- let RenderMethod = "addImmOperands";
+ let RenderMethod = "addS16ImmOperands";
}
def dispRIX : Operand<iPTR> {
let ParserMatchClass = PPCDispRIXOperand;
@@ -1030,7 +1030,7 @@ def dispRIHash : Operand<iPTR> {
}
def PPCDispRIX16Operand : AsmOperandClass {
let Name = "DispRIX16"; let PredicateMethod = "isS16ImmX16";
- let RenderMethod = "addImmOperands";
+ let RenderMethod = "addS16ImmOperands";
}
def dispRIX16 : Operand<iPTR> {
let ParserMatchClass = PPCDispRIX16Operand;
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 514789b3f645..4b940093482f 100644
--- a/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -583,10 +583,11 @@ void RISCVAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
bool RISCVAsmBackend::shouldInsertExtraNopBytesForCodeAlign(
const MCAlignFragment &AF, unsigned &Size) {
// Calculate Nops Size only when linker relaxation enabled.
- if (!STI.getFeatureBits()[RISCV::FeatureRelax])
+ const MCSubtargetInfo *STI = AF.getSubtargetInfo();
+ if (!STI->getFeatureBits()[RISCV::FeatureRelax])
return false;
- bool HasStdExtC = STI.getFeatureBits()[RISCV::FeatureStdExtC];
+ bool HasStdExtC = STI->getFeatureBits()[RISCV::FeatureStdExtC];
unsigned MinNopLen = HasStdExtC ? 2 : 4;
if (AF.getAlignment() <= MinNopLen) {
@@ -606,7 +607,8 @@ bool RISCVAsmBackend::shouldInsertFixupForCodeAlign(MCAssembler &Asm,
const MCAsmLayout &Layout,
MCAlignFragment &AF) {
// Insert the fixup only when linker relaxation enabled.
- if (!STI.getFeatureBits()[RISCV::FeatureRelax])
+ const MCSubtargetInfo *STI = AF.getSubtargetInfo();
+ if (!STI->getFeatureBits()[RISCV::FeatureRelax])
return false;
// Calculate total Nops we need to insert. If there are none to insert
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
index e935179e5f9b..4adcd25600f2 100644
--- a/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
@@ -302,32 +302,34 @@ InstSeq generateInstSeq(int64_t Val, const FeatureBitset &ActiveFeatures) {
TmpSeq.push_back(RISCVMatInt::Inst(Opc, 0));
if (TmpSeq.size() < Res.size())
Res = TmpSeq;
- }
- // Try to use LUI+SH*ADD+ADDI.
- int64_t Hi52 = ((uint64_t)Val + 0x800ull) & ~0xfffull;
- int64_t Lo12 = SignExtend64<12>(Val);
- Div = 0;
- if (isInt<32>(Hi52 / 3) && (Hi52 % 3) == 0) {
- Div = 3;
- Opc = RISCV::SH1ADD;
- } else if (isInt<32>(Hi52 / 5) && (Hi52 % 5) == 0) {
- Div = 5;
- Opc = RISCV::SH2ADD;
- } else if (isInt<32>(Hi52 / 9) && (Hi52 % 9) == 0) {
- Div = 9;
- Opc = RISCV::SH3ADD;
- }
- // Build the new instruction sequence.
- if (Div > 0) {
- // For Val that has zero Lo12 (implies Val equals to Hi52) should has
- // already been processed to LUI+SH*ADD by previous optimization.
- assert(Lo12 != 0 &&
- "unexpected instruction sequence for immediate materialisation");
- generateInstSeqImpl(Hi52 / Div, ActiveFeatures, TmpSeq);
- TmpSeq.push_back(RISCVMatInt::Inst(Opc, 0));
- TmpSeq.push_back(RISCVMatInt::Inst(RISCV::ADDI, Lo12));
- if (TmpSeq.size() < Res.size())
- Res = TmpSeq;
+ } else {
+ // Try to use LUI+SH*ADD+ADDI.
+ int64_t Hi52 = ((uint64_t)Val + 0x800ull) & ~0xfffull;
+ int64_t Lo12 = SignExtend64<12>(Val);
+ Div = 0;
+ if (isInt<32>(Hi52 / 3) && (Hi52 % 3) == 0) {
+ Div = 3;
+ Opc = RISCV::SH1ADD;
+ } else if (isInt<32>(Hi52 / 5) && (Hi52 % 5) == 0) {
+ Div = 5;
+ Opc = RISCV::SH2ADD;
+ } else if (isInt<32>(Hi52 / 9) && (Hi52 % 9) == 0) {
+ Div = 9;
+ Opc = RISCV::SH3ADD;
+ }
+ // Build the new instruction sequence.
+ if (Div > 0) {
+ // For Val that has zero Lo12 (implies Val equals to Hi52) should has
+ // already been processed to LUI+SH*ADD by previous optimization.
+ assert(Lo12 != 0 &&
+ "unexpected instruction sequence for immediate materialisation");
+ assert(TmpSeq.empty() && "Expected empty TmpSeq");
+ generateInstSeqImpl(Hi52 / Div, ActiveFeatures, TmpSeq);
+ TmpSeq.push_back(RISCVMatInt::Inst(Opc, 0));
+ TmpSeq.push_back(RISCVMatInt::Inst(RISCV::ADDI, Lo12));
+ if (TmpSeq.size() < Res.size())
+ Res = TmpSeq;
+ }
}
}
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index f3cc7d3fb46f..8f250eeb7248 100644
--- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -674,7 +674,10 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
if (hasBP(MF)) {
FrameReg = RISCVABI::getBPReg();
// |--------------------------| -- <-- FP
- // | callee-saved registers | | <----.
+ // | callee-allocated save | | <----|
+ // | area for register varargs| | |
+ // |--------------------------| | |
+ // | callee-saved registers | | |
// |--------------------------| -- |
// | realignment (the size of | | |
// | this area is not counted | | |
@@ -699,7 +702,10 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
} else {
FrameReg = RISCV::X2;
// |--------------------------| -- <-- FP
- // | callee-saved registers | | <----.
+ // | callee-allocated save | | <----|
+ // | area for register varargs| | |
+ // |--------------------------| | |
+ // | callee-saved registers | | |
// |--------------------------| -- |
// | realignment (the size of | | |
// | this area is not counted | | |
@@ -742,6 +748,9 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
// the frame size.
//
// |--------------------------| -- <-- FP
+ // | callee-allocated save | |
+ // | area for register varargs| |
+ // |--------------------------| |
// | callee-saved registers | |
// |--------------------------| | MFI.getStackSize()
// | scalar local variables | |
@@ -756,7 +765,10 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
// When using SP to access frame objects, we need to add RVV stack size.
//
// |--------------------------| -- <-- FP
- // | callee-saved registers | | <----.
+ // | callee-allocated save | | <----|
+ // | area for register varargs| | |
+ // |--------------------------| | |
+ // | callee-saved registers | | |
// |--------------------------| -- |
// | Padding after RVV | | |
// | (not counted in | | |
@@ -786,8 +798,11 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
Offset += StackOffset::getFixed(MFI.getStackSize());
}
} else if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
+ int ScalarLocalVarSize = MFI.getStackSize() -
+ RVFI->getCalleeSavedStackSize() -
+ RVFI->getVarArgsSaveSize();
Offset += StackOffset::get(
- alignTo(MFI.getStackSize() - RVFI->getCalleeSavedStackSize(), 8),
+ alignTo(ScalarLocalVarSize, 8),
RVFI->getRVVStackSize());
}
}
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e7672a7652cd..274b86593e0f 100644
--- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1908,37 +1908,27 @@ static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
// A zero-value value difference means that we're somewhere in the middle
// of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
// step change before evaluating the sequence.
- if (ValDiff != 0) {
- int64_t Remainder = ValDiff % IdxDiff;
- // Normalize the step if it's greater than 1.
- if (Remainder != ValDiff) {
- // The difference must cleanly divide the element span.
- if (Remainder != 0)
- return None;
- ValDiff /= IdxDiff;
- IdxDiff = 1;
- }
-
- if (!SeqStepNum)
- SeqStepNum = ValDiff;
- else if (ValDiff != SeqStepNum)
- return None;
+ if (ValDiff == 0)
+ continue;
- if (!SeqStepDenom)
- SeqStepDenom = IdxDiff;
- else if (IdxDiff != *SeqStepDenom)
+ int64_t Remainder = ValDiff % IdxDiff;
+ // Normalize the step if it's greater than 1.
+ if (Remainder != ValDiff) {
+ // The difference must cleanly divide the element span.
+ if (Remainder != 0)
return None;
+ ValDiff /= IdxDiff;
+ IdxDiff = 1;
}
- }
- // Record and/or check any addend.
- if (SeqStepNum && SeqStepDenom) {
- uint64_t ExpectedVal =
- (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
- int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
- if (!SeqAddend)
- SeqAddend = Addend;
- else if (SeqAddend != Addend)
+ if (!SeqStepNum)
+ SeqStepNum = ValDiff;
+ else if (ValDiff != SeqStepNum)
+ return None;
+
+ if (!SeqStepDenom)
+ SeqStepDenom = IdxDiff;
+ else if (IdxDiff != *SeqStepDenom)
return None;
}
@@ -1946,11 +1936,29 @@ static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
if (!PrevElt || PrevElt->first != Val)
PrevElt = std::make_pair(Val, Idx);
}
- // We need to have logged both a step and an addend for this to count as
- // a legal index sequence.
- if (!SeqStepNum || !SeqStepDenom || !SeqAddend)
+
+ // We need to have logged a step for this to count as a legal index sequence.
+ if (!SeqStepNum || !SeqStepDenom)
return None;
+ // Loop back through the sequence and validate elements we might have skipped
+ // while waiting for a valid step. While doing this, log any sequence addend.
+ for (unsigned Idx = 0; Idx < NumElts; Idx++) {
+ if (Op.getOperand(Idx).isUndef())
+ continue;
+ uint64_t Val = Op.getConstantOperandVal(Idx) &
+ maskTrailingOnes<uint64_t>(EltSizeInBits);
+ uint64_t ExpectedVal =
+ (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
+ int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
+ if (!SeqAddend)
+ SeqAddend = Addend;
+ else if (Addend != SeqAddend)
+ return None;
+ }
+
+ assert(SeqAddend && "Must have an addend if we have a step");
+
return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
}
@@ -2109,7 +2117,8 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
// a single addi instruction.
if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
(StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
- isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) {
+ isPowerOf2_32(StepDenominator) &&
+ (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
// Convert right out of the scalable type so we can use standard ISD
// nodes for the rest of the computation. If we used scalable types with
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
new file mode 100644
index 000000000000..bde0326a8de4
--- /dev/null
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
@@ -0,0 +1,30 @@
+//=- RISCVMachineFunctionInfo.cpp - RISCV machine function info ---*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares RISCV-specific per-machine-function information.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCVMachineFunctionInfo.h"
+
+using namespace llvm;
+
+yaml::RISCVMachineFunctionInfo::RISCVMachineFunctionInfo(
+ const llvm::RISCVMachineFunctionInfo &MFI)
+ : VarArgsFrameIndex(MFI.getVarArgsFrameIndex()),
+ VarArgsSaveSize(MFI.getVarArgsSaveSize()) {}
+
+void yaml::RISCVMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
+ MappingTraits<RISCVMachineFunctionInfo>::mapping(YamlIO, *this);
+}
+
+void RISCVMachineFunctionInfo::initializeBaseYamlFields(
+ const yaml::RISCVMachineFunctionInfo &YamlMFI) {
+ VarArgsFrameIndex = YamlMFI.VarArgsFrameIndex;
+ VarArgsSaveSize = YamlMFI.VarArgsSaveSize;
+}
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
index b5609e9a3890..a549ec211a94 100644
--- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -14,11 +14,34 @@
#define LLVM_LIB_TARGET_RISCV_RISCVMACHINEFUNCTIONINFO_H
#include "RISCVSubtarget.h"
+#include "llvm/CodeGen/MIRYamlMapping.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
namespace llvm {
+class RISCVMachineFunctionInfo;
+
+namespace yaml {
+struct RISCVMachineFunctionInfo final : public yaml::MachineFunctionInfo {
+ int VarArgsFrameIndex;
+ int VarArgsSaveSize;
+
+ RISCVMachineFunctionInfo() = default;
+ RISCVMachineFunctionInfo(const llvm::RISCVMachineFunctionInfo &MFI);
+
+ void mappingImpl(yaml::IO &YamlIO) override;
+ ~RISCVMachineFunctionInfo() = default;
+};
+
+template <> struct MappingTraits<RISCVMachineFunctionInfo> {
+ static void mapping(IO &YamlIO, RISCVMachineFunctionInfo &MFI) {
+ YamlIO.mapOptional("varArgsFrameIndex", MFI.VarArgsFrameIndex);
+ YamlIO.mapOptional("varArgsSaveSize", MFI.VarArgsSaveSize);
+ }
+};
+} // end namespace yaml
+
/// RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo
/// and contains private RISCV-specific information for each MachineFunction.
class RISCVMachineFunctionInfo : public MachineFunctionInfo {
@@ -74,6 +97,8 @@ public:
unsigned getCalleeSavedStackSize() const { return CalleeSavedStackSize; }
void setCalleeSavedStackSize(unsigned Size) { CalleeSavedStackSize = Size; }
+
+ void initializeBaseYamlFields(const yaml::RISCVMachineFunctionInfo &YamlMFI);
};
} // end namespace llvm
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index db5e2f1eeb6f..0248ea0039bc 100644
--- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -13,6 +13,7 @@
#include "RISCVTargetMachine.h"
#include "MCTargetDesc/RISCVBaseInfo.h"
#include "RISCV.h"
+#include "RISCVMachineFunctionInfo.h"
#include "RISCVTargetObjectFile.h"
#include "RISCVTargetTransformInfo.h"
#include "TargetInfo/RISCVTargetInfo.h"
@@ -22,6 +23,8 @@
#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
#include "llvm/CodeGen/GlobalISel/Legalizer.h"
#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
+#include "llvm/CodeGen/MIRParser/MIParser.h"
+#include "llvm/CodeGen/MIRYamlMapping.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -208,3 +211,23 @@ void RISCVPassConfig::addPreRegAlloc() {
addPass(createRISCVMergeBaseOffsetOptPass());
addPass(createRISCVInsertVSETVLIPass());
}
+
+yaml::MachineFunctionInfo *
+RISCVTargetMachine::createDefaultFuncInfoYAML() const {
+ return new yaml::RISCVMachineFunctionInfo();
+}
+
+yaml::MachineFunctionInfo *
+RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
+ const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
+ return new yaml::RISCVMachineFunctionInfo(*MFI);
+}
+
+bool RISCVTargetMachine::parseMachineFunctionInfo(
+ const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
+ SMDiagnostic &Error, SMRange &SourceRange) const {
+ const auto &YamlMFI =
+ static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
+ PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
+ return false;
+}
diff --git a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.h b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.h
index 3156333f7ee1..2a0212c846a0 100644
--- a/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.h
+++ b/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.h
@@ -46,6 +46,14 @@ public:
virtual bool isNoopAddrSpaceCast(unsigned SrcAS,
unsigned DstAS) const override;
+
+ yaml::MachineFunctionInfo *createDefaultFuncInfoYAML() const override;
+ yaml::MachineFunctionInfo *
+ convertFuncInfoToYAML(const MachineFunction &MF) const override;
+ bool parseMachineFunctionInfo(const yaml::MachineFunctionInfo &,
+ PerFunctionMIParsingState &PFS,
+ SMDiagnostic &Error,
+ SMRange &SourceRange) const override;
};
} // namespace llvm
diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp
index 77c2e7d16990..682932b8f3e6 100644
--- a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -15796,7 +15796,8 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
V1 = extract128BitVector(V1V2, 0, DAG, DL);
V2 = extract128BitVector(V1V2, 4, DAG, DL);
} else {
- SmallVector<SDValue> DWordClearOps(4, DAG.getConstant(0, DL, MVT::i32));
+ SmallVector<SDValue, 4> DWordClearOps(4,
+ DAG.getConstant(0, DL, MVT::i32));
for (unsigned i = 0; i != 4; i += 1 << (NumEvenDrops - 1))
DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32);
SDValue DWordClearMask =
@@ -47109,8 +47110,7 @@ static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
// into:
// srl(ctlz x), log2(bitsize(x))
// Input pattern is checked by caller.
-static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, EVT ExtTy,
- SelectionDAG &DAG) {
+static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) {
SDValue Cmp = Op.getOperand(1);
EVT VT = Cmp.getOperand(0).getValueType();
unsigned Log2b = Log2_32(VT.getSizeInBits());
@@ -47121,7 +47121,7 @@ static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, EVT ExtTy,
SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
DAG.getConstant(Log2b, dl, MVT::i8));
- return DAG.getZExtOrTrunc(Scc, dl, ExtTy);
+ return Scc;
}
// Try to transform:
@@ -47181,11 +47181,10 @@ static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
// or(srl(ctlz),srl(ctlz)).
// The dag combiner can then fold it into:
// srl(or(ctlz, ctlz)).
- EVT VT = OR->getValueType(0);
- SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, VT, DAG);
+ SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, DAG);
SDValue Ret, NewRHS;
- if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, VT, DAG)))
- Ret = DAG.getNode(ISD::OR, SDLoc(OR), VT, NewLHS, NewRHS);
+ if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG)))
+ Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS);
if (!Ret)
return SDValue();
@@ -47198,16 +47197,13 @@ static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
// Swap rhs with lhs to match or(setcc(eq, cmp, 0), or).
if (RHS->getOpcode() == ISD::OR)
std::swap(LHS, RHS);
- NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, VT, DAG);
+ NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG);
if (!NewRHS)
return SDValue();
- Ret = DAG.getNode(ISD::OR, SDLoc(OR), VT, Ret, NewRHS);
+ Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS);
}
- if (Ret)
- Ret = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
-
- return Ret;
+ return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
}
static SDValue foldMaskedMergeImpl(SDValue And0_L, SDValue And0_R,
diff --git a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 6bbb0251f2bc..2aab79e89078 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1961,6 +1961,12 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
}
}
+ // If this 'and' clears the sign-bits added by ashr, replace with lshr:
+ // and (ashr X, ShiftC), C --> lshr X, ShiftC
+ if (match(Op0, m_AShr(m_Value(X), m_APInt(ShiftC))) && ShiftC->ult(Width) &&
+ C->isMask(Width - ShiftC->getZExtValue()))
+ return BinaryOperator::CreateLShr(X, ConstantInt::get(Ty, *ShiftC));
+
const APInt *AddC;
if (match(Op0, m_Add(m_Value(X), m_APInt(AddC)))) {
// If we add zeros to every bit below a mask, the add has no effect:
diff --git a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 65e60498ff95..881b00f2a55a 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1572,6 +1572,23 @@ Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
}
}
+ // Canonicalize a signbit condition to use zero constant by swapping:
+ // (CmpLHS > -1) ? TV : FV --> (CmpLHS < 0) ? FV : TV
+ // To avoid conflicts (infinite loops) with other canonicalizations, this is
+ // not applied with any constant select arm.
+ if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes()) &&
+ !match(TrueVal, m_Constant()) && !match(FalseVal, m_Constant()) &&
+ ICI->hasOneUse()) {
+ InstCombiner::BuilderTy::InsertPointGuard Guard(Builder);
+ Builder.SetInsertPoint(&SI);
+ Value *IsNeg = Builder.CreateICmpSLT(
+ CmpLHS, ConstantInt::getNullValue(CmpLHS->getType()), ICI->getName());
+ replaceOperand(SI, 0, IsNeg);
+ SI.swapValues();
+ SI.swapProfMetadata();
+ return &SI;
+ }
+
// FIXME: This code is nearly duplicated in InstSimplify. Using/refactoring
// decomposeBitTestICmp() might help.
{
diff --git a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 21c16f07e237..46ff0994e04e 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2544,19 +2544,18 @@ void InnerLoopVectorizer::widenIntOrFpInduction(
Type *ScalarTy = IntegerType::get(ScalarIV->getContext(),
Step->getType()->getScalarSizeInBits());
- Instruction::BinaryOps IncOp = ID.getInductionOpcode();
- if (IncOp == Instruction::BinaryOpsEnd)
- IncOp = Instruction::Add;
for (unsigned Part = 0; Part < UF; ++Part) {
Value *StartIdx = ConstantInt::get(ScalarTy, Part);
- Instruction::BinaryOps MulOp = Instruction::Mul;
+ Value *EntryPart;
if (Step->getType()->isFloatingPointTy()) {
StartIdx = Builder.CreateUIToFP(StartIdx, Step->getType());
- MulOp = Instruction::FMul;
+ Value *MulOp = Builder.CreateFMul(StartIdx, Step);
+ EntryPart = Builder.CreateBinOp(ID.getInductionOpcode(), ScalarIV,
+ MulOp, "induction");
+ } else {
+ EntryPart = Builder.CreateAdd(
+ ScalarIV, Builder.CreateMul(StartIdx, Step), "induction");
}
-
- Value *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
- Value *EntryPart = Builder.CreateBinOp(IncOp, ScalarIV, Mul, "induction");
State.set(Def, EntryPart, Part);
if (Trunc) {
assert(!Step->getType()->isFloatingPointTy() &&
@@ -6035,6 +6034,18 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
!(InterleaveSmallLoopScalarReduction && HasReductions && VF.isScalar()))
return 1;
+ // If we did not calculate the cost for VF (because the user selected the VF)
+ // then we calculate the cost of VF here.
+ if (LoopCost == 0) {
+ InstructionCost C = expectedCost(VF).first;
+ assert(C.isValid() && "Expected to have chosen a VF with valid cost");
+ LoopCost = *C.getValue();
+
+ // Loop body is free and there is no need for interleaving.
+ if (LoopCost == 0)
+ return 1;
+ }
+
RegisterUsage R = calculateRegisterUsage({VF})[0];
// We divide by these constants so assume that we have at least one
// instruction that uses at least one register.
@@ -6126,16 +6137,6 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
assert(IC > 0 && "Interleave count must be greater than 0.");
- // If we did not calculate the cost for VF (because the user selected the VF)
- // then we calculate the cost of VF here.
- if (LoopCost == 0) {
- InstructionCost C = expectedCost(VF).first;
- assert(C.isValid() && "Expected to have chosen a VF with valid cost");
- LoopCost = *C.getValue();
- }
-
- assert(LoopCost && "Non-zero loop cost expected");
-
// Interleave if we vectorized this loop and there is a reduction that could
// benefit from interleaving.
if (VF.isVector() && HasReductions) {
diff --git a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 620d388199e0..258f6c67e54d 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -152,12 +152,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
- // If original AS != Load's AS, we can't bitcast the original pointer and have
- // to use Load's operand instead. Ideally we would want to strip pointer casts
- // without changing AS, but there's no API to do that ATM.
unsigned AS = Load->getPointerAddressSpace();
- if (AS != SrcPtr->getType()->getPointerAddressSpace())
- SrcPtr = Load->getPointerOperand();
// We are potentially transforming byte-sized (8-bit) memory accesses, so make
// sure we have all of our type-based constraints in place for this target.
@@ -245,7 +240,8 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
// It is safe and potentially profitable to load a vector directly:
// inselt undef, load Scalar, 0 --> load VecPtr
IRBuilder<> Builder(Load);
- Value *CastedPtr = Builder.CreateBitCast(SrcPtr, MinVecTy->getPointerTo(AS));
+ Value *CastedPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ SrcPtr, MinVecTy->getPointerTo(AS));
Value *VecLd = Builder.CreateAlignedLoad(MinVecTy, CastedPtr, Alignment);
VecLd = Builder.CreateShuffleVector(VecLd, Mask);
diff --git a/contrib/llvm-project/llvm/tools/llvm-objdump/ELFDump.cpp b/contrib/llvm-project/llvm/tools/llvm-objdump/ELFDump.cpp
index 98e71497d022..ca73dafe2b8e 100644
--- a/contrib/llvm-project/llvm/tools/llvm-objdump/ELFDump.cpp
+++ b/contrib/llvm-project/llvm/tools/llvm-objdump/ELFDump.cpp
@@ -171,8 +171,12 @@ uint64_t objdump::getELFSectionLMA(const object::ELFSectionRef &Sec) {
template <class ELFT>
static void printDynamicSection(const ELFFile<ELFT> &Elf, StringRef Filename) {
- ArrayRef<typename ELFT::Dyn> DynamicEntries =
- unwrapOrError(Elf.dynamicEntries(), Filename);
+ auto DynamicEntriesOrErr = Elf.dynamicEntries();
+ if (!DynamicEntriesOrErr) {
+ reportWarning(toString(DynamicEntriesOrErr.takeError()), Filename);
+ return;
+ }
+ ArrayRef<typename ELFT::Dyn> DynamicEntries = *DynamicEntriesOrErr;
// Find the maximum tag name length to format the value column properly.
size_t MaxLen = 0;