diff options
Diffstat (limited to 'contrib/llvm/lib')
27 files changed, 353 insertions, 100 deletions
diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index 8cabad4ad312..154f81f2622d 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -1836,7 +1836,10 @@ TypeIndex CodeViewDebug::lowerTypeMemberFunction(const DISubroutineType *Ty, unsigned Index = 0; SmallVector<TypeIndex, 8> ArgTypeIndices; - TypeIndex ReturnTypeIndex = getTypeIndex(ReturnAndArgs[Index++]); + TypeIndex ReturnTypeIndex = TypeIndex::Void(); + if (ReturnAndArgs.size() > Index) { + ReturnTypeIndex = getTypeIndex(ReturnAndArgs[Index++]); + } // If the first argument is a pointer type and this isn't a static method, // treat it as the special 'this' parameter, which is encoded separately from diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index 1de2ffb6cfa1..18c5fe27b1a8 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -1956,8 +1956,10 @@ void DebugLocEntry::finalize(const AsmPrinter &AP, void DwarfDebug::emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry) { // Emit the size. Asm->OutStreamer->AddComment("Loc expr size"); - Asm->emitInt16(DebugLocs.getBytes(Entry).size()); - + if (getDwarfVersion() >= 5) + Asm->EmitULEB128(DebugLocs.getBytes(Entry).size()); + else + Asm->emitInt16(DebugLocs.getBytes(Entry).size()); // Emit the entry. APByteStreamer Streamer(*Asm); emitDebugLocEntry(Streamer, Entry); diff --git a/contrib/llvm/lib/CodeGen/MachineInstr.cpp b/contrib/llvm/lib/CodeGen/MachineInstr.cpp index 764a84c7e132..dc1ad953e71d 100644 --- a/contrib/llvm/lib/CodeGen/MachineInstr.cpp +++ b/contrib/llvm/lib/CodeGen/MachineInstr.cpp @@ -225,12 +225,13 @@ void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) { } #ifndef NDEBUG - bool isMetaDataOp = Op.getType() == MachineOperand::MO_Metadata; + bool isDebugOp = Op.getType() == MachineOperand::MO_Metadata || + Op.getType() == MachineOperand::MO_MCSymbol; // OpNo now points as the desired insertion point. Unless this is a variadic // instruction, only implicit regs are allowed beyond MCID->getNumOperands(). // RegMask operands go between the explicit and implicit operands. assert((isImpReg || Op.isRegMask() || MCID->isVariadic() || - OpNo < MCID->getNumOperands() || isMetaDataOp) && + OpNo < MCID->getNumOperands() || isDebugOp) && "Trying to add an operand to a machine instr that is already done!"); #endif diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index ff5505c97721..6af01423ca10 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -16214,23 +16214,29 @@ static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) { // The build vector contains some number of undef elements and exactly // one other element. That other element must be a zero-extended scalar // extracted from a vector at a constant index to turn this into a shuffle. + // Also, require that the build vector does not implicitly truncate/extend + // its elements. // TODO: This could be enhanced to allow ANY_EXTEND as well as ZERO_EXTEND. + EVT VT = BV->getValueType(0); SDValue Zext = BV->getOperand(ZextElt); if (Zext.getOpcode() != ISD::ZERO_EXTEND || !Zext.hasOneUse() || Zext.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT || - !isa<ConstantSDNode>(Zext.getOperand(0).getOperand(1))) + !isa<ConstantSDNode>(Zext.getOperand(0).getOperand(1)) || + Zext.getValueSizeInBits() != VT.getScalarSizeInBits()) return SDValue(); - // The zero-extend must be a multiple of the source size. + // The zero-extend must be a multiple of the source size, and we must be + // building a vector of the same size as the source of the extract element. SDValue Extract = Zext.getOperand(0); unsigned DestSize = Zext.getValueSizeInBits(); unsigned SrcSize = Extract.getValueSizeInBits(); - if (DestSize % SrcSize != 0) + if (DestSize % SrcSize != 0 || + Extract.getOperand(0).getValueSizeInBits() != VT.getSizeInBits()) return SDValue(); // Create a shuffle mask that will combine the extracted element with zeros // and undefs. - int ZextRatio = DestSize / SrcSize; + int ZextRatio = DestSize / SrcSize; int NumMaskElts = NumBVOps * ZextRatio; SmallVector<int, 32> ShufMask(NumMaskElts, -1); for (int i = 0; i != NumMaskElts; ++i) { @@ -16260,7 +16266,7 @@ static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) { SDValue ZeroVec = DAG.getConstant(0, DL, VecVT); SDValue Shuf = DAG.getVectorShuffle(VecVT, DL, Extract.getOperand(0), ZeroVec, ShufMask); - return DAG.getBitcast(BV->getValueType(0), Shuf); + return DAG.getBitcast(VT, Shuf); } // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT diff --git a/contrib/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/contrib/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp index f8b5ff6ec8fb..94df6946f3ae 100644 --- a/contrib/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp +++ b/contrib/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp @@ -184,7 +184,8 @@ DWARFDebugLoclists::parseOneLocationList(DataExtractor Data, unsigned *Offset, } if (Kind != dwarf::DW_LLE_base_address) { - unsigned Bytes = Data.getU16(Offset); + unsigned Bytes = + Version >= 5 ? Data.getULEB128(Offset) : Data.getU16(Offset); // A single location description describing the location of the object... StringRef str = Data.getData().substr(*Offset, Bytes); *Offset += Bytes; diff --git a/contrib/llvm/lib/IR/AutoUpgrade.cpp b/contrib/llvm/lib/IR/AutoUpgrade.cpp index b2eb8b09982e..27064154221f 100644 --- a/contrib/llvm/lib/IR/AutoUpgrade.cpp +++ b/contrib/llvm/lib/IR/AutoUpgrade.cpp @@ -469,6 +469,11 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, } } + if (Name == "seh.recoverfp") { + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); + return true; + } + return false; } @@ -544,10 +549,6 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); return true; } - if (Name == "x86.seh.recoverfp") { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); - return true; - } break; } diff --git a/contrib/llvm/lib/Support/JSON.cpp b/contrib/llvm/lib/Support/JSON.cpp index d468013fb94a..07a556814915 100644 --- a/contrib/llvm/lib/Support/JSON.cpp +++ b/contrib/llvm/lib/Support/JSON.cpp @@ -182,6 +182,12 @@ bool operator==(const Value &L, const Value &R) { case Value::Boolean: return *L.getAsBoolean() == *R.getAsBoolean(); case Value::Number: + // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=323 + // The same integer must convert to the same double, per the standard. + // However we see 64-vs-80-bit precision comparisons with gcc-7 -O3 -m32. + // So we avoid floating point promotion for exact comparisons. + if (L.Type == Value::T_Integer || R.Type == Value::T_Integer) + return L.getAsInteger() == R.getAsInteger(); return *L.getAsNumber() == *R.getAsNumber(); case Value::String: return *L.getAsString() == *R.getAsString(); diff --git a/contrib/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp b/contrib/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp index e9699b0367d3..50300305abe3 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp @@ -103,6 +103,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/IR/DebugLoc.h" #include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" @@ -146,25 +147,31 @@ private: BitVector RegsAlreadyMasked; bool functionUsesHardeningRegister(MachineFunction &MF) const; - bool instrumentControlFlow(MachineBasicBlock &MBB); + bool instrumentControlFlow(MachineBasicBlock &MBB, + bool &UsesFullSpeculationBarrier); bool endsWithCondControlFlow(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, AArch64CC::CondCode &CondCode) const; void insertTrackingCode(MachineBasicBlock &SplitEdgeBB, AArch64CC::CondCode &CondCode, DebugLoc DL) const; - void insertSPToRegTaintPropagation(MachineBasicBlock *MBB, + void insertSPToRegTaintPropagation(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; - void insertRegToSPTaintPropagation(MachineBasicBlock *MBB, + void insertRegToSPTaintPropagation(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned TmpReg) const; + void insertFullSpeculationBarrier(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + DebugLoc DL) const; bool slhLoads(MachineBasicBlock &MBB); bool makeGPRSpeculationSafe(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineInstr &MI, unsigned Reg); - bool lowerSpeculationSafeValuePseudos(MachineBasicBlock &MBB); + bool lowerSpeculationSafeValuePseudos(MachineBasicBlock &MBB, + bool UsesFullSpeculationBarrier); bool expandSpeculationSafeValue(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI); + MachineBasicBlock::iterator MBBI, + bool UsesFullSpeculationBarrier); bool insertCSDB(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL); }; @@ -207,15 +214,19 @@ bool AArch64SpeculationHardening::endsWithCondControlFlow( return true; } +void AArch64SpeculationHardening::insertFullSpeculationBarrier( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + DebugLoc DL) const { + // A full control flow speculation barrier consists of (DSB SYS + ISB) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::DSB)).addImm(0xf); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ISB)).addImm(0xf); +} + void AArch64SpeculationHardening::insertTrackingCode( MachineBasicBlock &SplitEdgeBB, AArch64CC::CondCode &CondCode, DebugLoc DL) const { if (UseControlFlowSpeculationBarrier) { - // insert full control flow speculation barrier (DSB SYS + ISB) - BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::ISB)) - .addImm(0xf); - BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::DSB)) - .addImm(0xf); + insertFullSpeculationBarrier(SplitEdgeBB, SplitEdgeBB.begin(), DL); } else { BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::CSELXr)) .addDef(MisspeculatingTaintReg) @@ -227,7 +238,7 @@ void AArch64SpeculationHardening::insertTrackingCode( } bool AArch64SpeculationHardening::instrumentControlFlow( - MachineBasicBlock &MBB) { + MachineBasicBlock &MBB, bool &UsesFullSpeculationBarrier) { LLVM_DEBUG(dbgs() << "Instrument control flow tracking on MBB: " << MBB); bool Modified = false; @@ -263,55 +274,105 @@ bool AArch64SpeculationHardening::instrumentControlFlow( } // Perform correct code generation around function calls and before returns. - { - SmallVector<MachineInstr *, 4> ReturnInstructions; - SmallVector<MachineInstr *, 4> CallInstructions; + // The below variables record the return/terminator instructions and the call + // instructions respectively; including which register is available as a + // temporary register just before the recorded instructions. + SmallVector<std::pair<MachineInstr *, unsigned>, 4> ReturnInstructions; + SmallVector<std::pair<MachineInstr *, unsigned>, 4> CallInstructions; + // if a temporary register is not available for at least one of the + // instructions for which we need to transfer taint to the stack pointer, we + // need to insert a full speculation barrier. + // TmpRegisterNotAvailableEverywhere tracks that condition. + bool TmpRegisterNotAvailableEverywhere = false; + + RegScavenger RS; + RS.enterBasicBlock(MBB); + + for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); I++) { + MachineInstr &MI = *I; + if (!MI.isReturn() && !MI.isCall()) + continue; - for (MachineInstr &MI : MBB) { - if (MI.isReturn()) - ReturnInstructions.push_back(&MI); - else if (MI.isCall()) - CallInstructions.push_back(&MI); - } + // The RegScavenger represents registers available *after* the MI + // instruction pointed to by RS.getCurrentPosition(). + // We need to have a register that is available *before* the MI is executed. + if (I != MBB.begin()) + RS.forward(std::prev(I)); + // FIXME: The below just finds *a* unused register. Maybe code could be + // optimized more if this looks for the register that isn't used for the + // longest time around this place, to enable more scheduling freedom. Not + // sure if that would actually result in a big performance difference + // though. Maybe RegisterScavenger::findSurvivorBackwards has some logic + // already to do this - but it's unclear if that could easily be used here. + unsigned TmpReg = RS.FindUnusedReg(&AArch64::GPR64commonRegClass); + LLVM_DEBUG(dbgs() << "RS finds " + << ((TmpReg == 0) ? "no register " : "register "); + if (TmpReg != 0) dbgs() << printReg(TmpReg, TRI) << " "; + dbgs() << "to be available at MI " << MI); + if (TmpReg == 0) + TmpRegisterNotAvailableEverywhere = true; + if (MI.isReturn()) + ReturnInstructions.push_back({&MI, TmpReg}); + else if (MI.isCall()) + CallInstructions.push_back({&MI, TmpReg}); + } - Modified |= - (ReturnInstructions.size() > 0) || (CallInstructions.size() > 0); + if (TmpRegisterNotAvailableEverywhere) { + // When a temporary register is not available everywhere in this basic + // basic block where a propagate-taint-to-sp operation is needed, just + // emit a full speculation barrier at the start of this basic block, which + // renders the taint/speculation tracking in this basic block unnecessary. + insertFullSpeculationBarrier(MBB, MBB.begin(), + (MBB.begin())->getDebugLoc()); + UsesFullSpeculationBarrier = true; + Modified = true; + } else { + for (auto MI_Reg : ReturnInstructions) { + assert(MI_Reg.second != 0); + LLVM_DEBUG( + dbgs() + << " About to insert Reg to SP taint propagation with temp register " + << printReg(MI_Reg.second, TRI) + << " on instruction: " << *MI_Reg.first); + insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second); + Modified = true; + } - for (MachineInstr *Return : ReturnInstructions) - insertRegToSPTaintPropagation(Return->getParent(), Return, AArch64::X17); - for (MachineInstr *Call : CallInstructions) { + for (auto MI_Reg : CallInstructions) { + assert(MI_Reg.second != 0); + LLVM_DEBUG(dbgs() << " About to insert Reg to SP and back taint " + "propagation with temp register " + << printReg(MI_Reg.second, TRI) + << " around instruction: " << *MI_Reg.first); // Just after the call: - MachineBasicBlock::iterator i = Call; - i++; - insertSPToRegTaintPropagation(Call->getParent(), i); + insertSPToRegTaintPropagation( + MBB, std::next((MachineBasicBlock::iterator)MI_Reg.first)); // Just before the call: - insertRegToSPTaintPropagation(Call->getParent(), Call, AArch64::X17); + insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second); + Modified = true; } } - return Modified; } void AArch64SpeculationHardening::insertSPToRegTaintPropagation( - MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) const { + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { // If full control flow speculation barriers are used, emit a control flow // barrier to block potential miss-speculation in flight coming in to this // function. if (UseControlFlowSpeculationBarrier) { - // insert full control flow speculation barrier (DSB SYS + ISB) - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::DSB)).addImm(0xf); - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ISB)).addImm(0xf); + insertFullSpeculationBarrier(MBB, MBBI, DebugLoc()); return; } // CMP SP, #0 === SUBS xzr, SP, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::SUBSXri)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::SUBSXri)) .addDef(AArch64::XZR) .addUse(AArch64::SP) .addImm(0) .addImm(0); // no shift // CSETM x16, NE === CSINV x16, xzr, xzr, EQ - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::CSINVXr)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::CSINVXr)) .addDef(MisspeculatingTaintReg) .addUse(AArch64::XZR) .addUse(AArch64::XZR) @@ -319,7 +380,7 @@ void AArch64SpeculationHardening::insertSPToRegTaintPropagation( } void AArch64SpeculationHardening::insertRegToSPTaintPropagation( - MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned TmpReg) const { // If full control flow speculation barriers are used, there will not be // miss-speculation when returning from this function, and therefore, also @@ -328,19 +389,19 @@ void AArch64SpeculationHardening::insertRegToSPTaintPropagation( return; // mov Xtmp, SP === ADD Xtmp, SP, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) .addDef(TmpReg) .addUse(AArch64::SP) .addImm(0) .addImm(0); // no shift // and Xtmp, Xtmp, TaintReg === AND Xtmp, Xtmp, TaintReg, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ANDXrs)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ANDXrs)) .addDef(TmpReg, RegState::Renamable) .addUse(TmpReg, RegState::Kill | RegState::Renamable) .addUse(MisspeculatingTaintReg, RegState::Kill) .addImm(0); // mov SP, Xtmp === ADD SP, Xtmp, #0 - BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) .addDef(AArch64::SP) .addUse(TmpReg, RegState::Kill) .addImm(0) @@ -484,7 +545,8 @@ bool AArch64SpeculationHardening::slhLoads(MachineBasicBlock &MBB) { /// \brief If MBBI references a pseudo instruction that should be expanded /// here, do the expansion and return true. Otherwise return false. bool AArch64SpeculationHardening::expandSpeculationSafeValue( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + bool UsesFullSpeculationBarrier) { MachineInstr &MI = *MBBI; unsigned Opcode = MI.getOpcode(); bool Is64Bit = true; @@ -499,7 +561,7 @@ bool AArch64SpeculationHardening::expandSpeculationSafeValue( // Just remove the SpeculationSafe pseudo's if control flow // miss-speculation isn't happening because we're already inserting barriers // to guarantee that. - if (!UseControlFlowSpeculationBarrier) { + if (!UseControlFlowSpeculationBarrier && !UsesFullSpeculationBarrier) { unsigned DstReg = MI.getOperand(0).getReg(); unsigned SrcReg = MI.getOperand(1).getReg(); // Mark this register and all its aliasing registers as needing to be @@ -537,7 +599,7 @@ bool AArch64SpeculationHardening::insertCSDB(MachineBasicBlock &MBB, } bool AArch64SpeculationHardening::lowerSpeculationSafeValuePseudos( - MachineBasicBlock &MBB) { + MachineBasicBlock &MBB, bool UsesFullSpeculationBarrier) { bool Modified = false; RegsNeedingCSDBBeforeUse.reset(); @@ -572,15 +634,16 @@ bool AArch64SpeculationHardening::lowerSpeculationSafeValuePseudos( break; } - if (NeedToEmitBarrier) + if (NeedToEmitBarrier && !UsesFullSpeculationBarrier) Modified |= insertCSDB(MBB, MBBI, DL); - Modified |= expandSpeculationSafeValue(MBB, MBBI); + Modified |= + expandSpeculationSafeValue(MBB, MBBI, UsesFullSpeculationBarrier); MBBI = NMBBI; } - if (RegsNeedingCSDBBeforeUse.any()) + if (RegsNeedingCSDBBeforeUse.any() && !UsesFullSpeculationBarrier) Modified |= insertCSDB(MBB, MBBI, DL); return Modified; @@ -609,7 +672,7 @@ bool AArch64SpeculationHardening::runOnMachineFunction(MachineFunction &MF) { Modified |= slhLoads(MBB); } - // 2.a Add instrumentation code to function entry and exits. + // 2. Add instrumentation code to function entry and exits. LLVM_DEBUG( dbgs() << "***** AArch64SpeculationHardening - track control flow *****\n"); @@ -620,17 +683,15 @@ bool AArch64SpeculationHardening::runOnMachineFunction(MachineFunction &MF) { EntryBlocks.push_back(LPI.LandingPadBlock); for (auto Entry : EntryBlocks) insertSPToRegTaintPropagation( - Entry, Entry->SkipPHIsLabelsAndDebug(Entry->begin())); - - // 2.b Add instrumentation code to every basic block. - for (auto &MBB : MF) - Modified |= instrumentControlFlow(MBB); + *Entry, Entry->SkipPHIsLabelsAndDebug(Entry->begin())); - LLVM_DEBUG(dbgs() << "***** AArch64SpeculationHardening - Lowering " - "SpeculationSafeValue Pseudos *****\n"); - // Step 3: Lower SpeculationSafeValue pseudo instructions. - for (auto &MBB : MF) - Modified |= lowerSpeculationSafeValuePseudos(MBB); + // 3. Add instrumentation code to every basic block. + for (auto &MBB : MF) { + bool UsesFullSpeculationBarrier = false; + Modified |= instrumentControlFlow(MBB, UsesFullSpeculationBarrier); + Modified |= + lowerSpeculationSafeValuePseudos(MBB, UsesFullSpeculationBarrier); + } return Modified; } diff --git a/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index d2fed6861477..f10d100bfe11 100644 --- a/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -65,10 +65,7 @@ class MCInstrInfo; } // end namespace llvm -static cl::opt<bool> -EmitJalrReloc("mips-jalr-reloc", cl::Hidden, - cl::desc("MIPS: Emit R_{MICRO}MIPS_JALR relocation with jalr"), - cl::init(true)); +extern cl::opt<bool> EmitJalrReloc; namespace { diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp index 18d7dd99be34..2f2dd4e03c40 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp @@ -15,6 +15,13 @@ using namespace llvm; +// Note: this option is defined here to be visible from libLLVMMipsAsmParser +// and libLLVMMipsCodeGen +cl::opt<bool> +EmitJalrReloc("mips-jalr-reloc", cl::Hidden, + cl::desc("MIPS: Emit R_{MICRO}MIPS_JALR relocation with jalr"), + cl::init(true)); + namespace { static const MCPhysReg O32IntRegs[4] = {Mips::A0, Mips::A1, Mips::A2, Mips::A3}; diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h index a90db2384c46..ab8a6753eadc 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h @@ -89,7 +89,10 @@ namespace MipsII { MO_GOT_HI16, MO_GOT_LO16, MO_CALL_HI16, - MO_CALL_LO16 + MO_CALL_LO16, + + /// Helper operand used to generate R_MIPS_JALR + MO_JALR }; enum { diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp index f43a4d980f92..e3dcbaccfd08 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp @@ -614,8 +614,9 @@ getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups, llvm_unreachable("Unhandled fixup kind!"); break; case MipsMCExpr::MEK_DTPREL: - llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only"); - break; + // MEK_DTPREL is used for marking TLS DIEExpr only + // and contains a regular sub-expression. + return getExprOpValue(MipsExpr->getSubExpr(), Fixups, STI); case MipsMCExpr::MEK_CALL_HI16: FixupKind = Mips::fixup_Mips_CALL_HI16; break; diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp index 99857e083c6c..2d7312725205 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp @@ -44,8 +44,10 @@ void MipsMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const { llvm_unreachable("MEK_None and MEK_Special are invalid"); break; case MEK_DTPREL: - llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only"); - break; + // MEK_DTPREL is used for marking TLS DIEExpr only + // and contains a regular sub-expression. + getSubExpr()->print(OS, MAI, true); + return; case MEK_CALL_HI16: OS << "%call_hi"; break; @@ -161,7 +163,9 @@ MipsMCExpr::evaluateAsRelocatableImpl(MCValue &Res, case MEK_Special: llvm_unreachable("MEK_None and MEK_Special are invalid"); case MEK_DTPREL: - llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only"); + // MEK_DTPREL is used for marking TLS DIEExpr only + // and contains a regular sub-expression. + return getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup); case MEK_DTPREL_HI: case MEK_DTPREL_LO: case MEK_GOT: @@ -249,9 +253,6 @@ void MipsMCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const { case MEK_Special: llvm_unreachable("MEK_None and MEK_Special are invalid"); break; - case MEK_DTPREL: - llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only"); - break; case MEK_CALL_HI16: case MEK_CALL_LO16: case MEK_GOT: @@ -274,6 +275,7 @@ void MipsMCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const { if (const MipsMCExpr *E = dyn_cast<const MipsMCExpr>(getSubExpr())) E->fixELFSymbolsInTLSFixups(Asm); break; + case MEK_DTPREL: case MEK_DTPREL_HI: case MEK_DTPREL_LO: case MEK_TLSLDM: diff --git a/contrib/llvm/lib/Target/Mips/MicroMips32r6InstrInfo.td b/contrib/llvm/lib/Target/Mips/MicroMips32r6InstrInfo.td index 814918d25e70..c441aa76ad40 100644 --- a/contrib/llvm/lib/Target/Mips/MicroMips32r6InstrInfo.td +++ b/contrib/llvm/lib/Target/Mips/MicroMips32r6InstrInfo.td @@ -460,6 +460,7 @@ class JALRC16_MMR6_DESC_BASE<string opstr, RegisterOperand RO> let isCall = 1; let hasDelaySlot = 0; let Defs = [RA]; + let hasPostISelHook = 1; } class JALRC16_MMR6_DESC : JALRC16_MMR6_DESC_BASE<"jalr", GPR32Opnd>; diff --git a/contrib/llvm/lib/Target/Mips/MicroMipsInstrInfo.td b/contrib/llvm/lib/Target/Mips/MicroMipsInstrInfo.td index af380a0ec71e..ccc4f04bb92d 100644 --- a/contrib/llvm/lib/Target/Mips/MicroMipsInstrInfo.td +++ b/contrib/llvm/lib/Target/Mips/MicroMipsInstrInfo.td @@ -426,6 +426,7 @@ class JumpLinkRegMM16<string opstr, RegisterOperand RO> : let isCall = 1; let hasDelaySlot = 1; let Defs = [RA]; + let hasPostISelHook = 1; } // 16-bit Jump Reg diff --git a/contrib/llvm/lib/Target/Mips/Mips32r6InstrInfo.td b/contrib/llvm/lib/Target/Mips/Mips32r6InstrInfo.td index 2bd0cf2d59a6..fb239f572ef2 100644 --- a/contrib/llvm/lib/Target/Mips/Mips32r6InstrInfo.td +++ b/contrib/llvm/lib/Target/Mips/Mips32r6InstrInfo.td @@ -1105,7 +1105,7 @@ def : MipsPat<(select i32:$cond, immz, i32:$f), // Pseudo instructions let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, hasDelaySlot = 1, - hasExtraSrcRegAllocReq = 1, isCTI = 1, Defs = [AT] in { + hasExtraSrcRegAllocReq = 1, isCTI = 1, Defs = [AT], hasPostISelHook = 1 in { class TailCallRegR6<Instruction JumpInst, Register RT, RegisterOperand RO> : PseudoSE<(outs), (ins RO:$rs), [(MipsTailCall RO:$rs)], II_JR>, PseudoInstExpansion<(JumpInst RT:$rt, RO:$rs)>; diff --git a/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp b/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp index 362431fd42a6..a7a748b0840e 100644 --- a/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp +++ b/contrib/llvm/lib/Target/Mips/MipsAsmPrinter.cpp @@ -68,6 +68,8 @@ using namespace llvm; #define DEBUG_TYPE "mips-asm-printer" +extern cl::opt<bool> EmitJalrReloc; + MipsTargetStreamer &MipsAsmPrinter::getTargetStreamer() const { return static_cast<MipsTargetStreamer &>(*OutStreamer->getTargetStreamer()); } @@ -148,6 +150,40 @@ void MipsAsmPrinter::emitPseudoIndirectBranch(MCStreamer &OutStreamer, EmitToStreamer(OutStreamer, TmpInst0); } +// If there is an MO_JALR operand, insert: +// +// .reloc tmplabel, R_{MICRO}MIPS_JALR, symbol +// tmplabel: +// +// This is an optimization hint for the linker which may then replace +// an indirect call with a direct branch. +static void emitDirectiveRelocJalr(const MachineInstr &MI, + MCContext &OutContext, + TargetMachine &TM, + MCStreamer &OutStreamer, + const MipsSubtarget &Subtarget) { + for (unsigned int I = MI.getDesc().getNumOperands(), E = MI.getNumOperands(); + I < E; ++I) { + MachineOperand MO = MI.getOperand(I); + if (MO.isMCSymbol() && (MO.getTargetFlags() & MipsII::MO_JALR)) { + MCSymbol *Callee = MO.getMCSymbol(); + if (Callee && !Callee->getName().empty()) { + MCSymbol *OffsetLabel = OutContext.createTempSymbol(); + const MCExpr *OffsetExpr = + MCSymbolRefExpr::create(OffsetLabel, OutContext); + const MCExpr *CaleeExpr = + MCSymbolRefExpr::create(Callee, OutContext); + OutStreamer.EmitRelocDirective + (*OffsetExpr, + Subtarget.inMicroMipsMode() ? "R_MICROMIPS_JALR" : "R_MIPS_JALR", + CaleeExpr, SMLoc(), *TM.getMCSubtargetInfo()); + OutStreamer.EmitLabel(OffsetLabel); + return; + } + } + } +} + void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) { MipsTargetStreamer &TS = getTargetStreamer(); unsigned Opc = MI->getOpcode(); @@ -207,6 +243,11 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) { return; } + if (EmitJalrReloc && + (MI->isReturn() || MI->isCall() || MI->isIndirectBranch())) { + emitDirectiveRelocJalr(*MI, OutContext, TM, *OutStreamer, *Subtarget); + } + MachineBasicBlock::const_instr_iterator I = MI->getIterator(); MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); diff --git a/contrib/llvm/lib/Target/Mips/MipsFastISel.cpp b/contrib/llvm/lib/Target/Mips/MipsFastISel.cpp index 22ade31a72cd..a18416b9e861 100644 --- a/contrib/llvm/lib/Target/Mips/MipsFastISel.cpp +++ b/contrib/llvm/lib/Target/Mips/MipsFastISel.cpp @@ -56,6 +56,7 @@ #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" +#include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSymbol.h" @@ -75,6 +76,8 @@ using namespace llvm; +extern cl::opt<bool> EmitJalrReloc; + namespace { class MipsFastISel final : public FastISel { @@ -1551,6 +1554,16 @@ bool MipsFastISel::fastLowerCall(CallLoweringInfo &CLI) { CLI.Call = MIB; + if (EmitJalrReloc && !Subtarget->inMips16Mode()) { + // Attach callee address to the instruction, let asm printer emit + // .reloc R_MIPS_JALR. + if (Symbol) + MIB.addSym(Symbol, MipsII::MO_JALR); + else + MIB.addSym(FuncInfo.MF->getContext().getOrCreateSymbol( + Addr.getGlobalValue()->getName()), MipsII::MO_JALR); + } + // Finish off the call including any return values. return finishCall(CLI, RetVT, NumBytes); } diff --git a/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp b/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp index 8c2a364cdfa9..0f9c075ba0cc 100644 --- a/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/contrib/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -57,6 +57,7 @@ #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" +#include "llvm/MC/MCContext.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" @@ -91,6 +92,8 @@ NoZeroDivCheck("mno-check-zero-division", cl::Hidden, cl::desc("MIPS: Don't trap on integer division by zero."), cl::init(false)); +extern cl::opt<bool> EmitJalrReloc; + static const MCPhysReg Mips64DPRegs[8] = { Mips::D12_64, Mips::D13_64, Mips::D14_64, Mips::D15_64, Mips::D16_64, Mips::D17_64, Mips::D18_64, Mips::D19_64 @@ -2879,6 +2882,54 @@ getOpndList(SmallVectorImpl<SDValue> &Ops, Ops.push_back(InFlag); } +void MipsTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, + SDNode *Node) const { + switch (MI.getOpcode()) { + default: + return; + case Mips::JALR: + case Mips::JALRPseudo: + case Mips::JALR64: + case Mips::JALR64Pseudo: + case Mips::JALR16_MM: + case Mips::JALRC16_MMR6: + case Mips::TAILCALLREG: + case Mips::TAILCALLREG64: + case Mips::TAILCALLR6REG: + case Mips::TAILCALL64R6REG: + case Mips::TAILCALLREG_MM: + case Mips::TAILCALLREG_MMR6: { + if (!EmitJalrReloc || + Subtarget.inMips16Mode() || + !isPositionIndependent() || + Node->getNumOperands() < 1 || + Node->getOperand(0).getNumOperands() < 2) { + return; + } + // We are after the callee address, set by LowerCall(). + // If added to MI, asm printer will emit .reloc R_MIPS_JALR for the + // symbol. + const SDValue TargetAddr = Node->getOperand(0).getOperand(1); + StringRef Sym; + if (const GlobalAddressSDNode *G = + dyn_cast_or_null<const GlobalAddressSDNode>(TargetAddr)) { + Sym = G->getGlobal()->getName(); + } + else if (const ExternalSymbolSDNode *ES = + dyn_cast_or_null<const ExternalSymbolSDNode>(TargetAddr)) { + Sym = ES->getSymbol(); + } + + if (Sym.empty()) + return; + + MachineFunction *MF = MI.getParent()->getParent(); + MCSymbol *S = MF->getContext().getOrCreateSymbol(Sym); + MI.addOperand(MachineOperand::CreateMCSymbol(S, MipsII::MO_JALR)); + } + } +} + /// LowerCall - functions arguments are copied from virtual regs to /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted. SDValue @@ -2930,7 +2981,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // the maximum out going argument area (including the reserved area), and // preallocates the stack space on entrance to the caller. // - // FIXME: We should do the same for efficency and space. + // FIXME: We should do the same for efficiency and space. // Note: The check on the calling convention below must match // MipsABIInfo::GetCalleeAllocdArgSizeInBytes(). diff --git a/contrib/llvm/lib/Target/Mips/MipsISelLowering.h b/contrib/llvm/lib/Target/Mips/MipsISelLowering.h index e043f133a09f..c88633be02b7 100644 --- a/contrib/llvm/lib/Target/Mips/MipsISelLowering.h +++ b/contrib/llvm/lib/Target/Mips/MipsISelLowering.h @@ -341,6 +341,9 @@ class TargetRegisterClass; EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override; + void AdjustInstrPostInstrSelection(MachineInstr &MI, + SDNode *Node) const override; + void HandleByVal(CCState *, unsigned &, unsigned) const override; unsigned getRegisterByName(const char* RegName, EVT VT, diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp index bfb4c775205d..e38bef4663a7 100644 --- a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp +++ b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.cpp @@ -653,6 +653,16 @@ MipsInstrInfo::genInstrWithNewOpc(unsigned NewOpc, MIB.addImm(0); + // If I has an MCSymbol operand (used by asm printer, to emit R_MIPS_JALR), + // add it to the new instruction. + for (unsigned J = I->getDesc().getNumOperands(), E = I->getNumOperands(); + J < E; ++J) { + const MachineOperand &MO = I->getOperand(J); + if (MO.isMCSymbol() && (MO.getTargetFlags() & MipsII::MO_JALR)) + MIB.addSym(MO.getMCSymbol(), MipsII::MO_JALR); + } + + } else { for (unsigned J = 0, E = I->getDesc().getNumOperands(); J < E; ++J) { if (BranchWithZeroOperand && (unsigned)ZeroOperandPosition == J) @@ -825,7 +835,8 @@ MipsInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { {MO_GOT_HI16, "mips-got-hi16"}, {MO_GOT_LO16, "mips-got-lo16"}, {MO_CALL_HI16, "mips-call-hi16"}, - {MO_CALL_LO16, "mips-call-lo16"} + {MO_CALL_LO16, "mips-call-lo16"}, + {MO_JALR, "mips-jalr"} }; return makeArrayRef(Flags); } diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td index d9398b7d6024..46721e6cb9e5 100644 --- a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td +++ b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td @@ -1623,11 +1623,15 @@ let isCall=1, hasDelaySlot=1, isCTI=1, Defs = [RA] in { class JumpLinkRegPseudo<RegisterOperand RO, Instruction JALRInst, Register RetReg, RegisterOperand ResRO = RO>: PseudoSE<(outs), (ins RO:$rs), [(MipsJmpLink RO:$rs)], II_JALR>, - PseudoInstExpansion<(JALRInst RetReg, ResRO:$rs)>; + PseudoInstExpansion<(JALRInst RetReg, ResRO:$rs)> { + let hasPostISelHook = 1; + } class JumpLinkReg<string opstr, RegisterOperand RO>: InstSE<(outs RO:$rd), (ins RO:$rs), !strconcat(opstr, "\t$rd, $rs"), - [], II_JALR, FrmR, opstr>; + [], II_JALR, FrmR, opstr> { + let hasPostISelHook = 1; + } class BGEZAL_FT<string opstr, DAGOperand opnd, RegisterOperand RO> : @@ -1646,7 +1650,9 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, hasDelaySlot = 1, class TailCallReg<Instruction JumpInst, RegisterOperand RO> : PseudoSE<(outs), (ins RO:$rs), [(MipsTailCall RO:$rs)], II_JR>, - PseudoInstExpansion<(JumpInst RO:$rs)>; + PseudoInstExpansion<(JumpInst RO:$rs)> { + let hasPostISelHook = 1; + } } class BAL_BR_Pseudo<Instruction RealInst, DAGOperand opnd> : diff --git a/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp b/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp index 46b37ceae391..4a7c0ce2be19 100644 --- a/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp +++ b/contrib/llvm/lib/Target/Mips/MipsMCInstLower.cpp @@ -117,6 +117,8 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO, case MipsII::MO_CALL_LO16: TargetKind = MipsMCExpr::MEK_CALL_LO16; break; + case MipsII::MO_JALR: + return MCOperand(); } switch (MOTy) { diff --git a/contrib/llvm/lib/Target/X86/X86DiscriminateMemOps.cpp b/contrib/llvm/lib/Target/X86/X86DiscriminateMemOps.cpp index 3654bf04f4e9..6bee20b617dd 100644 --- a/contrib/llvm/lib/Target/X86/X86DiscriminateMemOps.cpp +++ b/contrib/llvm/lib/Target/X86/X86DiscriminateMemOps.cpp @@ -27,6 +27,14 @@ using namespace llvm; #define DEBUG_TYPE "x86-discriminate-memops" +static cl::opt<bool> EnableDiscriminateMemops( + DEBUG_TYPE, cl::init(false), + cl::desc("Generate unique debug info for each instruction with a memory " + "operand. Should be enabled for profile-drived cache prefetching, " + "both in the build of the binary being profiled, as well as in " + "the build of the binary consuming the profile."), + cl::Hidden); + namespace { using Location = std::pair<StringRef, unsigned>; @@ -67,6 +75,9 @@ char X86DiscriminateMemOps::ID = 0; X86DiscriminateMemOps::X86DiscriminateMemOps() : MachineFunctionPass(ID) {} bool X86DiscriminateMemOps::runOnMachineFunction(MachineFunction &MF) { + if (!EnableDiscriminateMemops) + return false; + DISubprogram *FDI = MF.getFunction().getSubprogram(); if (!FDI || !FDI->getUnit()->getDebugInfoForProfiling()) return false; diff --git a/contrib/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/contrib/llvm/lib/Target/X86/X86InsertPrefetch.cpp index 30b46a09ef0f..8bd57aa2278b 100644 --- a/contrib/llvm/lib/Target/X86/X86InsertPrefetch.cpp +++ b/contrib/llvm/lib/Target/X86/X86InsertPrefetch.cpp @@ -34,7 +34,8 @@ using namespace sampleprof; static cl::opt<std::string> PrefetchHintsFile("prefetch-hints-file", - cl::desc("Path to the prefetch hints profile."), + cl::desc("Path to the prefetch hints profile. See also " + "-x86-discriminate-memops"), cl::Hidden); namespace { diff --git a/contrib/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp b/contrib/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp index a9772e31da50..81d63ee80394 100644 --- a/contrib/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp +++ b/contrib/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp @@ -249,6 +249,8 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) { bool DoPromote = false; if (GV.hasLocalLinkage() && ((DoPromote = shouldPromoteLocalToGlobal(&GV)) || isPerformingImport())) { + // Save the original name string before we rename GV below. + auto Name = GV.getName().str(); // Once we change the name or linkage it is difficult to determine // again whether we should promote since shouldPromoteLocalToGlobal needs // to locate the summary (based on GUID from name and linkage). Therefore, @@ -257,6 +259,12 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) { GV.setLinkage(getLinkage(&GV, DoPromote)); if (!GV.hasLocalLinkage()) GV.setVisibility(GlobalValue::HiddenVisibility); + + // If we are renaming a COMDAT leader, ensure that we record the COMDAT + // for later renaming as well. This is required for COFF. + if (const auto *C = GV.getComdat()) + if (C->getName() == Name) + RenamedComdats.try_emplace(C, M.getOrInsertComdat(GV.getName())); } else GV.setLinkage(getLinkage(&GV, /* DoPromote */ false)); @@ -281,6 +289,16 @@ void FunctionImportGlobalProcessing::processGlobalsForThinLTO() { processGlobalForThinLTO(SF); for (GlobalAlias &GA : M.aliases()) processGlobalForThinLTO(GA); + + // Replace any COMDATS that required renaming (because the COMDAT leader was + // promoted and renamed). + if (!RenamedComdats.empty()) + for (auto &GO : M.global_objects()) + if (auto *C = GO.getComdat()) { + auto Replacement = RenamedComdats.find(C); + if (Replacement != RenamedComdats.end()) + GO.setComdat(Replacement->second); + } } bool FunctionImportGlobalProcessing::run() { diff --git a/contrib/llvm/lib/Transforms/Utils/LoopUtils.cpp b/contrib/llvm/lib/Transforms/Utils/LoopUtils.cpp index a93d1aeb62ef..112e80d27e34 100644 --- a/contrib/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/contrib/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -217,7 +217,10 @@ static Optional<bool> getOptionalBoolLoopAttribute(const Loop *TheLoop, // When the value is absent it is interpreted as 'attribute set'. return true; case 2: - return mdconst::extract_or_null<ConstantInt>(MD->getOperand(1).get()); + if (ConstantInt *IntMD = + mdconst::extract_or_null<ConstantInt>(MD->getOperand(1).get())) + return IntMD->getZExtValue(); + return true; } llvm_unreachable("unexpected number of options"); } @@ -376,17 +379,17 @@ TransformationMode llvm::hasVectorizeTransformation(Loop *L) { Optional<int> InterleaveCount = getOptionalIntLoopAttribute(L, "llvm.loop.interleave.count"); - if (Enable == true) { - // 'Forcing' vector width and interleave count to one effectively disables - // this tranformation. - if (VectorizeWidth == 1 && InterleaveCount == 1) - return TM_SuppressedByUser; - return TM_ForcedByUser; - } + // 'Forcing' vector width and interleave count to one effectively disables + // this tranformation. + if (Enable == true && VectorizeWidth == 1 && InterleaveCount == 1) + return TM_SuppressedByUser; if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized")) return TM_Disable; + if (Enable == true) + return TM_ForcedByUser; + if (VectorizeWidth == 1 && InterleaveCount == 1) return TM_Disable; |