diff options
Diffstat (limited to 'llvm/lib')
21 files changed, 299 insertions, 114 deletions
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp index bad2de9e5f5e..7ae7a1fd5493 100644 --- a/llvm/lib/Analysis/LazyValueInfo.cpp +++ b/llvm/lib/Analysis/LazyValueInfo.cpp @@ -96,9 +96,9 @@ static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B) { // Undefined is the strongest state. It means the value is known to be along // an unreachable path. - if (A.isUndefined()) + if (A.isUnknown()) return A; - if (B.isUndefined()) + if (B.isUnknown()) return B; // If we gave up for one, but got a useable fact from the other, use it. @@ -1203,7 +1203,7 @@ static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI, // false SETNE. if (isTrueDest == (Predicate == ICmpInst::ICMP_EQ)) return ValueLatticeElement::get(cast<Constant>(RHS)); - else + else if (!isa<UndefValue>(RHS)) return ValueLatticeElement::getNot(cast<Constant>(RHS)); } } @@ -1722,7 +1722,7 @@ ConstantRange LazyValueInfo::getConstantRange(Value *V, BasicBlock *BB, const DataLayout &DL = BB->getModule()->getDataLayout(); ValueLatticeElement Result = getImpl(PImpl, AC, &DL, DT).getValueInBlock(V, BB, CxtI); - if (Result.isUndefined()) + if (Result.isUnknown()) return ConstantRange::getEmpty(Width); if (Result.isConstantRange()) return Result.getConstantRange(); @@ -1761,7 +1761,7 @@ ConstantRange LazyValueInfo::getConstantRangeOnEdge(Value *V, ValueLatticeElement Result = getImpl(PImpl, AC, &DL, DT).getValueOnEdge(V, FromBB, ToBB, CxtI); - if (Result.isUndefined()) + if (Result.isUnknown()) return ConstantRange::getEmpty(Width); if (Result.isConstantRange()) return Result.getConstantRange(); @@ -1991,7 +1991,7 @@ void LazyValueInfoAnnotatedWriter::emitBasicBlockStartAnnot( for (auto &Arg : F->args()) { ValueLatticeElement Result = LVIImpl->getValueInBlock( const_cast<Argument *>(&Arg), const_cast<BasicBlock *>(BB)); - if (Result.isUndefined()) + if (Result.isUnknown()) continue; OS << "; LatticeVal for: '" << Arg << "' is: " << Result << "\n"; } diff --git a/llvm/lib/Analysis/ValueLattice.cpp b/llvm/lib/Analysis/ValueLattice.cpp index a0115a0eec36..eaf8885cc14e 100644 --- a/llvm/lib/Analysis/ValueLattice.cpp +++ b/llvm/lib/Analysis/ValueLattice.cpp @@ -10,8 +10,10 @@ namespace llvm { raw_ostream &operator<<(raw_ostream &OS, const ValueLatticeElement &Val) { - if (Val.isUndefined()) - return OS << "undefined"; + if (Val.isUnknown()) + return OS << "unknown"; + if (Val.isUndef()) + return OS << "undef"; if (Val.isOverdefined()) return OS << "overdefined"; diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp index a4ba197b7a1d..239b6fd6fd68 100644 --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -269,30 +269,26 @@ MachineSinking::AllUsesDominatedByBlock(unsigned Reg, // into and they are all PHI nodes. In this case, machine-sink must break // the critical edge first. e.g. // - // %bb.1: derived from LLVM BB %bb4.preheader + // %bb.1: // Predecessors according to CFG: %bb.0 // ... - // %reg16385 = DEC64_32r %reg16437, implicit-def dead %eflags + // %def = DEC64_32r %x, implicit-def dead %eflags // ... // JE_4 <%bb.37>, implicit %eflags // Successors according to CFG: %bb.37 %bb.2 // - // %bb.2: derived from LLVM BB %bb.nph - // Predecessors according to CFG: %bb.0 %bb.1 - // %reg16386 = PHI %reg16434, %bb.0, %reg16385, %bb.1 - BreakPHIEdge = true; - for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) { - MachineInstr *UseInst = MO.getParent(); - unsigned OpNo = &MO - &UseInst->getOperand(0); - MachineBasicBlock *UseBlock = UseInst->getParent(); - if (!(UseBlock == MBB && UseInst->isPHI() && - UseInst->getOperand(OpNo+1).getMBB() == DefMBB)) { - BreakPHIEdge = false; - break; - } - } - if (BreakPHIEdge) + // %bb.2: + // %p = PHI %y, %bb.0, %def, %bb.1 + if (llvm::all_of(MRI->use_nodbg_operands(Reg), [&](MachineOperand &MO) { + MachineInstr *UseInst = MO.getParent(); + unsigned OpNo = UseInst->getOperandNo(&MO); + MachineBasicBlock *UseBlock = UseInst->getParent(); + return UseBlock == MBB && UseInst->isPHI() && + UseInst->getOperand(OpNo + 1).getMBB() == DefMBB; + })) { + BreakPHIEdge = true; return true; + } for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) { // Determine the block of the use. diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 8ff04797c8d8..2476fd26f250 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -886,6 +886,13 @@ static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) { ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()); } +// Determine if this an indexed load with an opaque target constant index. +static bool canSplitIdx(LoadSDNode *LD) { + return MaySplitLoadIndex && + (LD->getOperand(2).getOpcode() != ISD::TargetConstant || + !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque()); +} + bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc, const SDLoc &DL, SDValue N0, @@ -14222,11 +14229,11 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) { auto ReplaceLd = [&](LoadSDNode *LD, SDValue Val, SDValue Chain) -> SDValue { if (LD->isIndexed()) { - bool IsSub = (LD->getAddressingMode() == ISD::PRE_DEC || - LD->getAddressingMode() == ISD::POST_DEC); - unsigned Opc = IsSub ? ISD::SUB : ISD::ADD; - SDValue Idx = DAG.getNode(Opc, SDLoc(LD), LD->getOperand(1).getValueType(), - LD->getOperand(1), LD->getOperand(2)); + // Cannot handle opaque target constants and we must respect the user's + // request not to split indexes from loads. + if (!canSplitIdx(LD)) + return SDValue(); + SDValue Idx = SplitIndexingFromLoad(LD); SDValue Ops[] = {Val, Idx, Chain}; return CombineTo(LD, Ops, 3); } @@ -14322,14 +14329,12 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) { // the indexing into an add/sub directly (that TargetConstant may not be // valid for a different type of node, and we cannot convert an opaque // target constant into a regular constant). - bool HasOTCInc = LD->getOperand(2).getOpcode() == ISD::TargetConstant && - cast<ConstantSDNode>(LD->getOperand(2))->isOpaque(); + bool CanSplitIdx = canSplitIdx(LD); - if (!N->hasAnyUseOfValue(0) && - ((MaySplitLoadIndex && !HasOTCInc) || !N->hasAnyUseOfValue(1))) { + if (!N->hasAnyUseOfValue(0) && (CanSplitIdx || !N->hasAnyUseOfValue(1))) { SDValue Undef = DAG.getUNDEF(N->getValueType(0)); SDValue Index; - if (N->hasAnyUseOfValue(1) && MaySplitLoadIndex && !HasOTCInc) { + if (N->hasAnyUseOfValue(1) && CanSplitIdx) { Index = SplitIndexingFromLoad(LD); // Try to fold the base pointer arithmetic into subsequent loads and // stores. diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 8294591b7326..6ecde9b43c07 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -225,6 +225,21 @@ static bool isRegUsedByPhiNodes(unsigned DefReg, return false; } +static bool isTerminatingEHLabel(MachineBasicBlock *MBB, MachineInstr &MI) { + // Ignore non-EH labels. + if (!MI.isEHLabel()) + return false; + + // Any EH label outside a landing pad must be for an invoke. Consider it a + // terminator. + if (!MBB->isEHPad()) + return true; + + // If this is a landingpad, the first non-phi instruction will be an EH_LABEL. + // Don't consider that label to be a terminator. + return MI.getIterator() != MBB->getFirstNonPHI(); +} + /// Build a map of instruction orders. Return the first terminator and its /// order. Consider EH_LABEL instructions to be terminators as well, since local /// values for phis after invokes must be materialized before the call. @@ -233,7 +248,7 @@ void FastISel::InstOrderMap::initialize( unsigned Order = 0; for (MachineInstr &I : *MBB) { if (!FirstTerminator && - (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) { + (I.isTerminator() || isTerminatingEHLabel(MBB, I))) { FirstTerminator = &I; FirstTerminatorOrder = Order; } diff --git a/llvm/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp b/llvm/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp index 4d7cd468f3ee..6924b0e0ca02 100644 --- a/llvm/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp +++ b/llvm/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp @@ -90,7 +90,9 @@ static inline ArrayRef<uint8_t> stabilize(BumpPtrAllocator &Alloc, TypeIndex MergingTypeTableBuilder::insertRecordAs(hash_code Hash, ArrayRef<uint8_t> &Record) { assert(Record.size() < UINT32_MAX && "Record too big"); - assert(Record.size() % 4 == 0 && "Record is not aligned to 4 bytes!"); + assert(Record.size() % 4 == 0 && + "The type record size is not a multiple of 4 bytes which will cause " + "misalignment in the output TPI stream!"); LocallyHashedType WeakHash{Hash, Record}; auto Result = HashedRecords.try_emplace(WeakHash, nextTypeIndex()); diff --git a/llvm/lib/DebugInfo/CodeView/TypeStreamMerger.cpp b/llvm/lib/DebugInfo/CodeView/TypeStreamMerger.cpp index f9fca74a2199..c233db5c1d06 100644 --- a/llvm/lib/DebugInfo/CodeView/TypeStreamMerger.cpp +++ b/llvm/lib/DebugInfo/CodeView/TypeStreamMerger.cpp @@ -360,16 +360,18 @@ Error TypeStreamMerger::remapType(const CVType &Type) { [this, Type](MutableArrayRef<uint8_t> Storage) -> ArrayRef<uint8_t> { return remapIndices(Type, Storage); }; + unsigned AlignedSize = alignTo(Type.RecordData.size(), 4); + if (LLVM_LIKELY(UseGlobalHashes)) { GlobalTypeTableBuilder &Dest = isIdRecord(Type.kind()) ? *DestGlobalIdStream : *DestGlobalTypeStream; GloballyHashedType H = GlobalHashes[CurIndex.toArrayIndex()]; - DestIdx = Dest.insertRecordAs(H, Type.RecordData.size(), DoSerialize); + DestIdx = Dest.insertRecordAs(H, AlignedSize, DoSerialize); } else { MergingTypeTableBuilder &Dest = isIdRecord(Type.kind()) ? *DestIdStream : *DestTypeStream; - RemapStorage.resize(Type.RecordData.size()); + RemapStorage.resize(AlignedSize); ArrayRef<uint8_t> Result = DoSerialize(RemapStorage); if (!Result.empty()) DestIdx = Dest.insertRecordBytes(Result); @@ -386,9 +388,15 @@ Error TypeStreamMerger::remapType(const CVType &Type) { ArrayRef<uint8_t> TypeStreamMerger::remapIndices(const CVType &OriginalType, MutableArrayRef<uint8_t> Storage) { + unsigned Align = OriginalType.RecordData.size() & 3; + unsigned AlignedSize = alignTo(OriginalType.RecordData.size(), 4); + assert(Storage.size() == AlignedSize && + "The storage buffer size is not a multiple of 4 bytes which will " + "cause misalignment in the output TPI stream!"); + SmallVector<TiReference, 4> Refs; discoverTypeIndices(OriginalType.RecordData, Refs); - if (Refs.empty()) + if (Refs.empty() && Align == 0) return OriginalType.RecordData; ::memcpy(Storage.data(), OriginalType.RecordData.data(), @@ -408,6 +416,16 @@ TypeStreamMerger::remapIndices(const CVType &OriginalType, return {}; } } + + if (Align > 0) { + RecordPrefix *StorageHeader = + reinterpret_cast<RecordPrefix *>(Storage.data()); + StorageHeader->RecordLen += 4 - Align; + + DestContent = Storage.data() + OriginalType.RecordData.size(); + for (; Align < 4; ++Align) + *DestContent++ = LF_PAD4 - Align; + } return Storage; } diff --git a/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp index 4f10f8524a9b..51a1f0a544e3 100644 --- a/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp @@ -44,6 +44,9 @@ void TpiStreamBuilder::setVersionHeader(PdbRaw_TpiVer Version) { void TpiStreamBuilder::addTypeRecord(ArrayRef<uint8_t> Record, Optional<uint32_t> Hash) { // If we just crossed an 8KB threshold, add a type index offset. + assert(((Record.size() & 3) == 0) && + "The type record's size is not a multiple of 4 bytes which will " + "cause misalignment in the output TPI stream!"); size_t NewSize = TypeRecordBytes + Record.size(); constexpr size_t EightKB = 8 * 1024; if (NewSize / EightKB > TypeRecordBytes / EightKB || TypeRecords.empty()) { @@ -153,8 +156,11 @@ Error TpiStreamBuilder::commit(const msf::MSFLayout &Layout, return EC; for (auto Rec : TypeRecords) { - assert(!Rec.empty()); // An empty record will not write anything, but it - // would shift all offsets from here on. + assert(!Rec.empty() && "Attempting to write an empty type record shifts " + "all offsets in the TPI stream!"); + assert(((Rec.size() & 3) == 0) && + "The type record's size is not a multiple of 4 bytes which will " + "cause misalignment in the output TPI stream!"); if (auto EC = Writer.writeBytes(Rec)) return EC; } diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp index d567cc14a830..8cbe09f040be 100644 --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -761,7 +761,6 @@ void MCObjectFileInfo::initWasmMCObjectFileInfo(const Triple &T) { Ctx->getWasmSection(".debug_ranges", SectionKind::getMetadata()); DwarfMacinfoSection = Ctx->getWasmSection(".debug_macinfo", SectionKind::getMetadata()); - DwarfAddrSection = Ctx->getWasmSection(".debug_addr", SectionKind::getMetadata()); DwarfCUIndexSection = Ctx->getWasmSection(".debug_cu_index", SectionKind::getMetadata()); DwarfTUIndexSection = Ctx->getWasmSection(".debug_tu_index", SectionKind::getMetadata()); DwarfInfoSection = @@ -770,6 +769,17 @@ void MCObjectFileInfo::initWasmMCObjectFileInfo(const Triple &T) { DwarfPubNamesSection = Ctx->getWasmSection(".debug_pubnames", SectionKind::getMetadata()); DwarfPubTypesSection = Ctx->getWasmSection(".debug_pubtypes", SectionKind::getMetadata()); + DwarfDebugNamesSection = + Ctx->getWasmSection(".debug_names", SectionKind::getMetadata()); + DwarfStrOffSection = + Ctx->getWasmSection(".debug_str_offsets", SectionKind::getMetadata()); + DwarfAddrSection = + Ctx->getWasmSection(".debug_addr", SectionKind::getMetadata()); + DwarfRnglistsSection = + Ctx->getWasmSection(".debug_rnglists", SectionKind::getMetadata()); + DwarfLoclistsSection = + Ctx->getWasmSection(".debug_loclists", SectionKind::getMetadata()); + // Wasm use data section for LSDA. // TODO Consider putting each function's exception table in a separate // section, as in -function-sections, to facilitate lld's --gc-section. diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 66f3f418d06c..9f504b1eaa42 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -4549,7 +4549,7 @@ SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op, static SDValue LowerSADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) { EVT VT = Op.getValueType(); - if (!Subtarget->hasDSP()) + if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) return SDValue(); if (!VT.isSimple()) return SDValue(); diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp index 062d1d36c43c..be02da18fb7d 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp @@ -116,26 +116,22 @@ const MCFixupKindInfo &ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const { // ARMFixupKinds.h. // // Name Offset (bits) Size (bits) Flags - {"fixup_arm_ldst_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, + {"fixup_arm_ldst_pcrel_12", 0, 32, IsPCRelConstant}, {"fixup_t2_ldst_pcrel_12", 0, 32, - MCFixupKindInfo::FKF_IsPCRel | - MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, - {"fixup_arm_pcrel_10_unscaled", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, - {"fixup_arm_pcrel_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, + IsPCRelConstant | MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, + {"fixup_arm_pcrel_10_unscaled", 0, 32, IsPCRelConstant}, + {"fixup_arm_pcrel_10", 0, 32, IsPCRelConstant}, {"fixup_t2_pcrel_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, {"fixup_arm_pcrel_9", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, {"fixup_t2_pcrel_9", 0, 32, - MCFixupKindInfo::FKF_IsPCRel | - MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, + IsPCRelConstant | MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, {"fixup_thumb_adr_pcrel_10", 8, 8, - MCFixupKindInfo::FKF_IsPCRel | - MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, - {"fixup_arm_adr_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, + IsPCRelConstant | MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, + {"fixup_arm_adr_pcrel_12", 0, 32, IsPCRelConstant}, {"fixup_t2_adr_pcrel_12", 0, 32, - MCFixupKindInfo::FKF_IsPCRel | - MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, + IsPCRelConstant | MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}, {"fixup_arm_condbranch", 8, 24, MCFixupKindInfo::FKF_IsPCRel}, {"fixup_arm_uncondbranch", 8, 24, MCFixupKindInfo::FKF_IsPCRel}, {"fixup_t2_condbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, diff --git a/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp b/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp index 29abc9303a62..b2ecb531db9d 100644 --- a/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp +++ b/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp @@ -116,11 +116,22 @@ void BPFMISimplifyPatchable::checkADDrr(MachineRegisterInfo *MRI, else continue; - // It must be a form of %1 = *(type *)(%2 + 0) or *(type *)(%2 + 0) = %1. + // It must be a form of %2 = *(type *)(%1 + 0) or *(type *)(%1 + 0) = %2. const MachineOperand &ImmOp = DefInst->getOperand(2); if (!ImmOp.isImm() || ImmOp.getImm() != 0) continue; + // Reject the form: + // %1 = ADD_rr %2, %3 + // *(type *)(%2 + 0) = %1 + if (Opcode == BPF::STB || Opcode == BPF::STH || Opcode == BPF::STW || + Opcode == BPF::STD || Opcode == BPF::STB32 || Opcode == BPF::STH32 || + Opcode == BPF::STW32) { + const MachineOperand &Opnd = DefInst->getOperand(0); + if (Opnd.isReg() && Opnd.getReg() == I->getReg()) + continue; + } + BuildMI(*DefInst->getParent(), *DefInst, DefInst->getDebugLoc(), TII->get(COREOp)) .add(DefInst->getOperand(0)).addImm(Opcode).add(*BaseOp) .addGlobalAddress(GVal); diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 60ed72e1018b..00f59bba52e8 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -7720,15 +7720,17 @@ void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, // Emit a store to the stack slot. SDValue Chain; + unsigned Alignment = DAG.getEVTAlignment(Tmp.getValueType()); if (i32Stack) { MachineFunction &MF = DAG.getMachineFunction(); + Alignment = 4; MachineMemOperand *MMO = - MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4); + MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Alignment); SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); } else - Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, MPI); + Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, MPI, Alignment); // Result is a load from the stack slot. If loading 4 bytes, make sure to // add in a bias on big endian. @@ -7741,6 +7743,7 @@ void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, RLI.Chain = Chain; RLI.Ptr = FIPtr; RLI.MPI = MPI; + RLI.Alignment = Alignment; } /// Custom lowers floating point to integer conversions to use @@ -7848,9 +7851,10 @@ bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, SelectionDAG &DAG, ISD::LoadExtType ET) const { SDLoc dl(Op); + bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT && + (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32); if (ET == ISD::NON_EXTLOAD && - (Op.getOpcode() == ISD::FP_TO_UINT || - Op.getOpcode() == ISD::FP_TO_SINT) && + (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) && isOperationLegalOrCustom(Op.getOpcode(), Op.getOperand(0).getValueType())) { @@ -13588,7 +13592,7 @@ SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, (Op1VT == MVT::i32 || Op1VT == MVT::i64 || (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8))); - if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Altivec() || + if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Vector() || cast<StoreSDNode>(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt) return SDValue(); diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index cbdd7135de43..60eefbc677da 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -33998,6 +33998,7 @@ static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops, return SDValue(); // Shuffle the constant bits according to the mask. + SDLoc DL(Root); APInt UndefElts(NumMaskElts, 0); APInt ZeroElts(NumMaskElts, 0); APInt ConstantElts(NumMaskElts, 0); @@ -34035,6 +34036,10 @@ static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops, } assert((UndefElts | ZeroElts | ConstantElts).isAllOnesValue()); + // Attempt to create a zero vector. + if ((UndefElts | ZeroElts).isAllOnesValue()) + return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, DL); + // Create the constant data. MVT MaskSVT; if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64)) @@ -34043,8 +34048,9 @@ static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops, MaskSVT = MVT::getIntegerVT(MaskSizeInBits); MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts); + if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) + return SDValue(); - SDLoc DL(Root); SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL); return DAG.getBitcast(VT, CstOp); } diff --git a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp index 48d0d8a35704..0a79b793a980 100644 --- a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp +++ b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp @@ -18,6 +18,7 @@ #include "X86.h" #include "X86InstrInfo.h" #include "X86Subtarget.h" +#include "X86TargetMachine.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -102,7 +103,16 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) { // Check that the cf-protection-branch is enabled. Metadata *isCFProtectionSupported = MF.getMMI().getModule()->getModuleFlag("cf-protection-branch"); - if (!isCFProtectionSupported && !IndirectBranchTracking) + // NB: We need to enable IBT in jitted code if JIT compiler is CET + // enabled. + const X86TargetMachine *TM = + static_cast<const X86TargetMachine *>(&MF.getTarget()); +#ifdef __CET__ + bool isJITwithCET = TM->isJIT(); +#else + bool isJITwithCET = false; +#endif + if (!isCFProtectionSupported && !IndirectBranchTracking && !isJITwithCET) return false; // True if the current MF was changed and false otherwise. @@ -111,10 +121,11 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) { TII = SubTarget.getInstrInfo(); EndbrOpcode = SubTarget.is64Bit() ? X86::ENDBR64 : X86::ENDBR32; - // Non-internal function or function whose address was taken, can be - // accessed through indirect calls. Mark the first BB with ENDBR instruction - // unless nocf_check attribute is used. - if ((MF.getFunction().hasAddressTaken() || + // Large code model, non-internal function or function whose address + // was taken, can be accessed through indirect calls. Mark the first + // BB with ENDBR instruction unless nocf_check attribute is used. + if ((TM->getCodeModel() == CodeModel::Large || + MF.getFunction().hasAddressTaken() || !MF.getFunction().hasLocalLinkage()) && !MF.getFunction().doesNoCfCheck()) { auto MBB = MF.begin(); @@ -127,11 +138,18 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) { if (MBB.hasAddressTaken()) Changed |= addENDBR(MBB, MBB.begin()); + // Exception handle may indirectly jump to catch pad, So we should add + // ENDBR before catch pad instructions. + bool EHPadIBTNeeded = MBB.isEHPad(); + for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) { - if (!I->isCall()) - continue; - if (IsCallReturnTwice(I->getOperand(0))) + if (I->isCall() && IsCallReturnTwice(I->getOperand(0))) + Changed |= addENDBR(MBB, std::next(I)); + + if (EHPadIBTNeeded && I->isEHLabel()) { Changed |= addENDBR(MBB, std::next(I)); + EHPadIBTNeeded = false; + } } } return Changed; diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 8c696e9adbed..7176e46f07b1 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -222,7 +222,7 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, getEffectiveRelocModel(TT, JIT, RM), getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64), OL), - TLOF(createTLOF(getTargetTriple())) { + TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) { // On PS4, the "return address" of a 'noreturn' call must still be within // the calling function, and TrapUnreachable is an easy way to get that. if (TT.isPS4() || TT.isOSBinFormatMachO()) { diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h index ec3db7b1e9e8..757ce8bc5c72 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -30,6 +30,8 @@ class TargetTransformInfo; class X86TargetMachine final : public LLVMTargetMachine { std::unique_ptr<TargetLoweringObjectFile> TLOF; mutable StringMap<std::unique_ptr<X86Subtarget>> SubtargetMap; + // True if this is used in JIT. + bool IsJIT; public: X86TargetMachine(const Target &T, const Triple &TT, StringRef CPU, @@ -52,6 +54,8 @@ public: TargetLoweringObjectFile *getObjFileLowering() const override { return TLOF.get(); } + + bool isJIT() const { return IsJIT; } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 66cb3e74e53e..1e067a45d016 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -1155,7 +1155,10 @@ static void simplifySuspendPoints(coro::Shape &Shape) { if (N == 0) return; while (true) { - if (simplifySuspendPoint(cast<CoroSuspendInst>(S[I]), Shape.CoroBegin)) { + auto SI = cast<CoroSuspendInst>(S[I]); + // Leave final.suspend to handleFinalSuspend since it is undefined behavior + // to resume a coroutine suspended at the final suspend point. + if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) { if (--N == I) break; std::swap(S[I], S[N]); diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp index 0fd966457ece..b75e853553c5 100644 --- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -450,14 +450,19 @@ static bool CanDoGlobalSRA(GlobalVariable *GV) { /// Copy over the debug info for a variable to its SRA replacements. static void transferSRADebugInfo(GlobalVariable *GV, GlobalVariable *NGV, uint64_t FragmentOffsetInBits, - uint64_t FragmentSizeInBits, - unsigned NumElements) { + uint64_t FragmentSizeInBits) { SmallVector<DIGlobalVariableExpression *, 1> GVs; GV->getDebugInfo(GVs); for (auto *GVE : GVs) { DIVariable *Var = GVE->getVariable(); + Optional<uint64_t> VarSize = Var->getSizeInBits(); + DIExpression *Expr = GVE->getExpression(); - if (NumElements > 1) { + // If the FragmentSize is smaller than the variable, + // emit a fragment expression. + // If the variable size is unknown a fragment must be + // emitted to be safe. + if (!VarSize || FragmentSizeInBits < *VarSize) { if (auto E = DIExpression::createFragmentExpression( Expr, FragmentOffsetInBits, FragmentSizeInBits)) Expr = *E; @@ -539,8 +544,7 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) { // Copy over the debug info for the variable. uint64_t Size = DL.getTypeAllocSizeInBits(NGV->getValueType()); uint64_t FragmentOffsetInBits = Layout.getElementOffsetInBits(ElementIdx); - transferSRADebugInfo(GV, NGV, FragmentOffsetInBits, Size, - STy->getNumElements()); + transferSRADebugInfo(GV, NGV, FragmentOffsetInBits, Size); } else if (SequentialType *STy = dyn_cast<SequentialType>(Ty)) { uint64_t EltSize = DL.getTypeAllocSize(ElTy); Align EltAlign(DL.getABITypeAlignment(ElTy)); @@ -553,7 +557,7 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) { if (NewAlign > EltAlign) NGV->setAlignment(NewAlign); transferSRADebugInfo(GV, NGV, FragmentSizeInBits * ElementIdx, - FragmentSizeInBits, STy->getNumElements()); + FragmentSizeInBits); } } diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp index bf3e4ed3e31f..8fd9e7a2e610 100644 --- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp @@ -115,7 +115,8 @@ private: // list. Function * insertCounterWriteout(ArrayRef<std::pair<GlobalVariable *, MDNode *>>); - Function *insertFlush(ArrayRef<std::pair<GlobalVariable *, MDNode *>>); + Function *insertReset(ArrayRef<std::pair<GlobalVariable *, MDNode *>>); + Function *insertFlush(Function *ResetF); void AddFlushBeforeForkAndExec(); @@ -630,35 +631,76 @@ static bool shouldKeepInEntry(BasicBlock::iterator It) { } void GCOVProfiler::AddFlushBeforeForkAndExec() { - SmallVector<Instruction *, 2> ForkAndExecs; + SmallVector<CallInst *, 2> Forks; + SmallVector<CallInst *, 2> Execs; for (auto &F : M->functions()) { auto *TLI = &GetTLI(F); for (auto &I : instructions(F)) { if (CallInst *CI = dyn_cast<CallInst>(&I)) { if (Function *Callee = CI->getCalledFunction()) { LibFunc LF; - if (TLI->getLibFunc(*Callee, LF) && - (LF == LibFunc_fork || LF == LibFunc_execl || - LF == LibFunc_execle || LF == LibFunc_execlp || - LF == LibFunc_execv || LF == LibFunc_execvp || - LF == LibFunc_execve || LF == LibFunc_execvpe || - LF == LibFunc_execvP)) { - ForkAndExecs.push_back(&I); + if (TLI->getLibFunc(*Callee, LF)) { + if (LF == LibFunc_fork) { +#if !defined(_WIN32) + Forks.push_back(CI); +#endif + } else if (LF == LibFunc_execl || LF == LibFunc_execle || + LF == LibFunc_execlp || LF == LibFunc_execv || + LF == LibFunc_execvp || LF == LibFunc_execve || + LF == LibFunc_execvpe || LF == LibFunc_execvP) { + Execs.push_back(CI); + } } } } } } - // We need to split the block after the fork/exec call - // because else the counters for the lines after will be - // the same as before the call. - for (auto I : ForkAndExecs) { - IRBuilder<> Builder(I); + for (auto F : Forks) { + IRBuilder<> Builder(F); + BasicBlock *Parent = F->getParent(); + auto NextInst = ++F->getIterator(); + + // We've a fork so just reset the counters in the child process + FunctionType *FTy = FunctionType::get(Builder.getInt32Ty(), {}, false); + FunctionCallee GCOVFork = M->getOrInsertFunction("__gcov_fork", FTy); + F->setCalledFunction(GCOVFork); + + // We split just after the fork to have a counter for the lines after + // Anyway there's a bug: + // void foo() { fork(); } + // void bar() { foo(); blah(); } + // then "blah();" will be called 2 times but showed as 1 + // because "blah()" belongs to the same block as "foo();" + Parent->splitBasicBlock(NextInst); + + // back() is a br instruction with a debug location + // equals to the one from NextAfterFork + // So to avoid to have two debug locs on two blocks just change it + DebugLoc Loc = F->getDebugLoc(); + Parent->back().setDebugLoc(Loc); + } + + for (auto E : Execs) { + IRBuilder<> Builder(E); + BasicBlock *Parent = E->getParent(); + auto NextInst = ++E->getIterator(); + + // Since the process is replaced by a new one we need to write out gcdas + // No need to reset the counters since they'll be lost after the exec** FunctionType *FTy = FunctionType::get(Builder.getVoidTy(), {}, false); - FunctionCallee GCOVFlush = M->getOrInsertFunction("__gcov_flush", FTy); - Builder.CreateCall(GCOVFlush); - I->getParent()->splitBasicBlock(I); + FunctionCallee WriteoutF = + M->getOrInsertFunction("llvm_writeout_files", FTy); + Builder.CreateCall(WriteoutF); + + DebugLoc Loc = E->getDebugLoc(); + Builder.SetInsertPoint(&*NextInst); + // If the exec** fails we must reset the counters since they've been + // dumped + FunctionCallee ResetF = M->getOrInsertFunction("llvm_reset_counters", FTy); + Builder.CreateCall(ResetF)->setDebugLoc(Loc); + Parent->splitBasicBlock(NextInst); + Parent->back().setDebugLoc(Loc); } } @@ -850,7 +892,8 @@ bool GCOVProfiler::emitProfileArcs() { } Function *WriteoutF = insertCounterWriteout(CountersBySP); - Function *FlushF = insertFlush(CountersBySP); + Function *ResetF = insertReset(CountersBySP); + Function *FlushF = insertFlush(ResetF); // Create a small bit of code that registers the "__llvm_gcov_writeout" to // be executed at exit and the "__llvm_gcov_flush" function to be executed @@ -868,16 +911,14 @@ bool GCOVProfiler::emitProfileArcs() { IRBuilder<> Builder(BB); FTy = FunctionType::get(Type::getVoidTy(*Ctx), false); - Type *Params[] = { - PointerType::get(FTy, 0), - PointerType::get(FTy, 0) - }; + Type *Params[] = {PointerType::get(FTy, 0), PointerType::get(FTy, 0), + PointerType::get(FTy, 0)}; FTy = FunctionType::get(Builder.getVoidTy(), Params, false); - // Initialize the environment and register the local writeout and flush - // functions. + // Initialize the environment and register the local writeout, flush and + // reset functions. FunctionCallee GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy); - Builder.CreateCall(GCOVInit, {WriteoutF, FlushF}); + Builder.CreateCall(GCOVInit, {WriteoutF, FlushF, ResetF}); Builder.CreateRetVoid(); appendToGlobalCtors(*M, F, 0); @@ -1190,8 +1231,43 @@ Function *GCOVProfiler::insertCounterWriteout( return WriteoutF; } -Function *GCOVProfiler:: -insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> > CountersBySP) { +Function *GCOVProfiler::insertReset( + ArrayRef<std::pair<GlobalVariable *, MDNode *>> CountersBySP) { + FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false); + Function *ResetF = M->getFunction("__llvm_gcov_reset"); + if (!ResetF) + ResetF = Function::Create(FTy, GlobalValue::InternalLinkage, + "__llvm_gcov_reset", M); + else + ResetF->setLinkage(GlobalValue::InternalLinkage); + ResetF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + ResetF->addFnAttr(Attribute::NoInline); + if (Options.NoRedZone) + ResetF->addFnAttr(Attribute::NoRedZone); + + BasicBlock *Entry = BasicBlock::Create(*Ctx, "entry", ResetF); + IRBuilder<> Builder(Entry); + + // Zero out the counters. + for (const auto &I : CountersBySP) { + GlobalVariable *GV = I.first; + Constant *Null = Constant::getNullValue(GV->getValueType()); + Builder.CreateStore(Null, GV); + } + + Type *RetTy = ResetF->getReturnType(); + if (RetTy->isVoidTy()) + Builder.CreateRetVoid(); + else if (RetTy->isIntegerTy()) + // Used if __llvm_gcov_reset was implicitly declared. + Builder.CreateRet(ConstantInt::get(RetTy, 0)); + else + report_fatal_error("invalid return type for __llvm_gcov_reset"); + + return ResetF; +} + +Function *GCOVProfiler::insertFlush(Function *ResetF) { FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false); Function *FlushF = M->getFunction("__llvm_gcov_flush"); if (!FlushF) @@ -1212,16 +1288,10 @@ insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> > CountersBySP) { IRBuilder<> Builder(Entry); Builder.CreateCall(WriteoutF, {}); - - // Zero out the counters. - for (const auto &I : CountersBySP) { - GlobalVariable *GV = I.first; - Constant *Null = Constant::getNullValue(GV->getValueType()); - Builder.CreateStore(Null, GV); - } + Builder.CreateCall(ResetF, {}); Type *RetTy = FlushF->getReturnType(); - if (RetTy == Type::getVoidTy(*Ctx)) + if (RetTy->isVoidTy()) Builder.CreateRetVoid(); else if (RetTy->isIntegerTy()) // Used if __llvm_gcov_flush was implicitly declared. diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp index 623a8b711ed8..ac53ff33e836 100644 --- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp +++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp @@ -104,6 +104,21 @@ static bool mergeEmptyReturnBlocks(Function &F) { continue; } + // Skip merging if this would result in a CallBr instruction with a + // duplicate destination. FIXME: See note in CodeGenPrepare.cpp. + bool SkipCallBr = false; + for (pred_iterator PI = pred_begin(&BB), E = pred_end(&BB); + PI != E && !SkipCallBr; ++PI) { + if (auto *CBI = dyn_cast<CallBrInst>((*PI)->getTerminator())) + for (unsigned i = 0, e = CBI->getNumSuccessors(); i != e; ++i) + if (RetBlock == CBI->getSuccessor(i)) { + SkipCallBr = true; + break; + } + } + if (SkipCallBr) + continue; + // Otherwise, we found a duplicate return block. Merge the two. Changed = true; |