diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine')
30 files changed, 1621 insertions, 766 deletions
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp index c741fe2b3778..ff1e8050c7e7 100644 --- a/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -183,13 +183,13 @@ LLVMBool LLVMCreateMCJITCompilerForModule( std::unique_ptr<Module> Mod(unwrap(M)); if (Mod) - // Set function attribute "no-frame-pointer-elim" based on + // Set function attribute "frame-pointer" based on // NoFramePointerElim. for (auto &F : *Mod) { auto Attrs = F.getAttributes(); - StringRef Value(options.NoFramePointerElim ? "true" : "false"); + StringRef Value = options.NoFramePointerElim ? "all" : "none"; Attrs = Attrs.addAttribute(F.getContext(), AttributeList::FunctionIndex, - "no-frame-pointer-elim", Value); + "frame-pointer", Value); F.setAttributes(Attrs); } diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index f80b0e7f8909..f1114e92c360 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -17,174 +17,281 @@ namespace llvm { namespace jitlink { -EHFrameBinaryParser::EHFrameBinaryParser(JITTargetAddress EHFrameAddress, - StringRef EHFrameContent, - unsigned PointerSize, - support::endianness Endianness) - : EHFrameAddress(EHFrameAddress), EHFrameContent(EHFrameContent), - PointerSize(PointerSize), EHFrameReader(EHFrameContent, Endianness) {} +EHFrameSplitter::EHFrameSplitter(StringRef EHFrameSectionName) + : EHFrameSectionName(EHFrameSectionName) {} -Error EHFrameBinaryParser::addToGraph() { - while (!EHFrameReader.empty()) { - size_t RecordOffset = EHFrameReader.getOffset(); +Error EHFrameSplitter::operator()(LinkGraph &G) { + auto *EHFrame = G.findSectionByName(EHFrameSectionName); + if (!EHFrame) { LLVM_DEBUG({ - dbgs() << "Processing eh-frame record at " - << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) - << " (offset " << RecordOffset << ")\n"; + dbgs() << "EHFrameSplitter: No " << EHFrameSectionName + << " section. Nothing to do\n"; }); + return Error::success(); + } - size_t RecordLength = 0; - uint32_t RecordLengthField; - if (auto Err = EHFrameReader.readInteger(RecordLengthField)) + LLVM_DEBUG({ + dbgs() << "EHFrameSplitter: Processing " << EHFrameSectionName << "...\n"; + }); + + DenseMap<Block *, LinkGraph::SplitBlockCache> Caches; + + { + // Pre-build the split caches. + for (auto *B : EHFrame->blocks()) + Caches[B] = LinkGraph::SplitBlockCache::value_type(); + for (auto *Sym : EHFrame->symbols()) + Caches[&Sym->getBlock()]->push_back(Sym); + for (auto *B : EHFrame->blocks()) + llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getOffset() > RHS->getOffset(); + }); + } + + // Iterate over blocks (we do this by iterating over Caches entries rather + // than EHFrame->blocks() as we will be inserting new blocks along the way, + // which would invalidate iterators in the latter sequence. + for (auto &KV : Caches) { + auto &B = *KV.first; + auto &BCache = KV.second; + if (auto Err = processBlock(G, B, BCache)) return Err; + } - // Process CIE/FDE length/extended-length fields to build the blocks. - // - // The value of these fields describe the length of the *rest* of the CIE - // (not including data up to the end of the field itself) so we have to - // bump RecordLength to include the data up to the end of the field: 4 bytes - // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. - if (RecordLengthField == 0) // Length 0 means end of __eh_frame section. - break; + return Error::success(); +} - // If the regular length field's value is 0xffffffff, use extended length. - if (RecordLengthField == 0xffffffff) { - uint64_t ExtendedLengthField; - if (auto Err = EHFrameReader.readInteger(ExtendedLengthField)) - return Err; - if (ExtendedLengthField > EHFrameReader.bytesRemaining()) - return make_error<JITLinkError>("CIE record extends past the end of " - "the __eh_frame section"); - if (ExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) - return make_error<JITLinkError>("CIE record too large to process"); - RecordLength = ExtendedLengthField + 12; - } else { - if (RecordLengthField > EHFrameReader.bytesRemaining()) - return make_error<JITLinkError>("CIE record extends past the end of " - "the __eh_frame section"); - RecordLength = RecordLengthField + 4; - } +Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, + LinkGraph::SplitBlockCache &Cache) { + LLVM_DEBUG({ + dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) + << "\n"; + }); - LLVM_DEBUG(dbgs() << " length: " << RecordLength << "\n"); + // eh-frame should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error<JITLinkError>("Unexpected zero-fill block in " + + EHFrameSectionName + " section"); - // Read the CIE Pointer. - size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); - uint32_t CIEPointer; - if (auto Err = EHFrameReader.readInteger(CIEPointer)) - return Err; + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + BinaryStreamReader BlockReader(B.getContent(), G.getEndianness()); - // Based on the CIE pointer value, parse this as a CIE or FDE record. - if (CIEPointer == 0) { - if (auto Err = processCIE(RecordOffset, RecordLength)) + while (true) { + uint64_t RecordStartOffset = BlockReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress()) << "\n"; + }); + + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + if (Length != 0xffffffff) { + if (auto Err = BlockReader.skip(Length)) return Err; } else { - if (auto Err = processFDE(RecordOffset, RecordLength, CIEPointerAddress, - CIEPointer)) + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + if (auto Err = BlockReader.skip(ExtendedLength)) return Err; } - EHFrameReader.setOffset(RecordOffset + RecordLength); + // If this was the last block then there's nothing to split + if (BlockReader.empty()) { + LLVM_DEBUG(dbgs() << " Extracted " << B << "\n"); + return Error::success(); + } + + uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; + auto &NewBlock = G.splitBlock(B, BlockSize); + (void)NewBlock; + LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); + } +} + +EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName, + Edge::Kind FDEToCIE, Edge::Kind FDEToPCBegin, + Edge::Kind FDEToLSDA) + : EHFrameSectionName(EHFrameSectionName), FDEToCIE(FDEToCIE), + FDEToPCBegin(FDEToPCBegin), FDEToLSDA(FDEToLSDA) {} + +Error EHFrameEdgeFixer::operator()(LinkGraph &G) { + auto *EHFrame = G.findSectionByName(EHFrameSectionName); + + if (!EHFrame) { + LLVM_DEBUG({ + dbgs() << "EHFrameEdgeFixer: No " << EHFrameSectionName + << " section. Nothing to do\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "EHFrameEdgeFixer: Processing " << EHFrameSectionName << "...\n"; + }); + + ParseContext PC(G); + + // Build a map of all blocks and symbols in the text sections. We will use + // these for finding / building edge targets when processing FDEs. + for (auto &Sec : G.sections()) { + PC.AddrToSyms.addSymbols(Sec.symbols()); + if (auto Err = PC.AddrToBlock.addBlocks(Sec.blocks(), + BlockAddressMap::includeNonNull)) + return Err; } + // Sort eh-frame blocks into address order to ensure we visit CIEs before + // their child FDEs. + std::vector<Block *> EHFrameBlocks; + for (auto *B : EHFrame->blocks()) + EHFrameBlocks.push_back(B); + llvm::sort(EHFrameBlocks, [](const Block *LHS, const Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Loop over the blocks in address order. + for (auto *B : EHFrameBlocks) + if (auto Err = processBlock(PC, *B)) + return Err; + return Error::success(); } -void EHFrameBinaryParser::anchor() {} +Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { -Expected<EHFrameBinaryParser::AugmentationInfo> -EHFrameBinaryParser::parseAugmentationString() { - AugmentationInfo AugInfo; - uint8_t NextChar; - uint8_t *NextField = &AugInfo.Fields[0]; + LLVM_DEBUG({ + dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) + << "\n"; + }); - if (auto Err = EHFrameReader.readInteger(NextChar)) - return std::move(Err); + // eh-frame should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error<JITLinkError>("Unexpected zero-fill block in " + + EHFrameSectionName + " section"); - while (NextChar != 0) { - switch (NextChar) { - case 'z': - AugInfo.AugmentationDataPresent = true; - break; - case 'e': - if (auto Err = EHFrameReader.readInteger(NextChar)) - return std::move(Err); - if (NextChar != 'h') - return make_error<JITLinkError>("Unrecognized substring e" + - Twine(NextChar) + - " in augmentation string"); - AugInfo.EHDataFieldPresent = true; - break; - case 'L': - case 'P': - case 'R': - *NextField++ = NextChar; - break; - default: - return make_error<JITLinkError>("Unrecognized character " + - Twine(NextChar) + - " in augmentation string"); + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + // Find the offsets of any existing edges from this block. + BlockEdgeMap BlockEdges; + for (auto &E : B.edges()) + if (E.isRelocation()) { + if (BlockEdges.count(E.getOffset())) + return make_error<JITLinkError>( + "Multiple relocations at offset " + + formatv("{0:x16}", E.getOffset()) + " in " + EHFrameSectionName + + " block at address " + formatv("{0:x16}", B.getAddress())); + + BlockEdges[E.getOffset()] = EdgeTarget(E); } - if (auto Err = EHFrameReader.readInteger(NextChar)) - return std::move(Err); - } + CIEInfosMap CIEInfos; + BinaryStreamReader BlockReader(B.getContent(), PC.G.getEndianness()); + while (!BlockReader.empty()) { + size_t RecordStartOffset = BlockReader.getOffset(); - return std::move(AugInfo); -} + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress() + RecordStartOffset) << "\n"; + }); -Expected<JITTargetAddress> EHFrameBinaryParser::readAbsolutePointer() { - static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), - "Result must be able to hold a uint64_t"); - JITTargetAddress Addr; - if (PointerSize == 8) { - if (auto Err = EHFrameReader.readInteger(Addr)) - return std::move(Err); - } else if (PointerSize == 4) { - uint32_t Addr32; - if (auto Err = EHFrameReader.readInteger(Addr32)) - return std::move(Err); - Addr = Addr32; - } else - llvm_unreachable("Pointer size is not 32-bit or 64-bit"); - return Addr; + // Get the record length. + size_t RecordRemaining; + { + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + // If Length < 0xffffffff then use the regular length field, otherwise + // read the extended length field. + if (Length != 0xffffffff) + RecordRemaining = Length; + else { + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + RecordRemaining = ExtendedLength; + } + } + + if (BlockReader.bytesRemaining() < RecordRemaining) + return make_error<JITLinkError>( + "Incomplete CFI record at " + + formatv("{0:x16}", B.getAddress() + RecordStartOffset)); + + // Read the CIE delta for this record. + uint64_t CIEDeltaFieldOffset = BlockReader.getOffset() - RecordStartOffset; + uint32_t CIEDelta; + if (auto Err = BlockReader.readInteger(CIEDelta)) + return Err; + + if (CIEDelta == 0) { + if (auto Err = processCIE(PC, B, RecordStartOffset, + CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset)) + return Err; + } else { + if (auto Err = processFDE(PC, B, RecordStartOffset, + CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset, CIEDelta, BlockEdges)) + return Err; + } + + // Move to the next record. + BlockReader.setOffset(RecordStartOffset + CIEDeltaFieldOffset + + RecordRemaining); + } + + return Error::success(); } -Error EHFrameBinaryParser::processCIE(size_t RecordOffset, - size_t RecordLength) { - // Use the dwarf namespace for convenient access to pointer encoding - // constants. +Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, + size_t RecordOffset, size_t RecordLength, + size_t CIEDeltaFieldOffset) { using namespace dwarf; - LLVM_DEBUG(dbgs() << " Record is CIE\n"); + LLVM_DEBUG(dbgs() << " Record is CIE\n"); - auto &CIESymbol = - createCIERecord(EHFrameAddress + RecordOffset, - EHFrameContent.substr(RecordOffset, RecordLength)); + auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); + BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); + + // Skip past the CIE delta field: we've already processed this far. + RecordReader.setOffset(CIEDeltaFieldOffset + 4); + auto &CIESymbol = + PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); CIEInformation CIEInfo(CIESymbol); uint8_t Version = 0; - if (auto Err = EHFrameReader.readInteger(Version)) + if (auto Err = RecordReader.readInteger(Version)) return Err; if (Version != 0x01) return make_error<JITLinkError>("Bad CIE version " + Twine(Version) + " (should be 0x01) in eh-frame"); - auto AugInfo = parseAugmentationString(); + auto AugInfo = parseAugmentationString(RecordReader); if (!AugInfo) return AugInfo.takeError(); // Skip the EH Data field if present. if (AugInfo->EHDataFieldPresent) - if (auto Err = EHFrameReader.skip(PointerSize)) + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) return Err; // Read and sanity check the code alignment factor. { uint64_t CodeAlignmentFactor = 0; - if (auto Err = EHFrameReader.readULEB128(CodeAlignmentFactor)) + if (auto Err = RecordReader.readULEB128(CodeAlignmentFactor)) return Err; if (CodeAlignmentFactor != 1) return make_error<JITLinkError>("Unsupported CIE code alignment factor " + @@ -195,7 +302,7 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, // Read and sanity check the data alignment factor. { int64_t DataAlignmentFactor = 0; - if (auto Err = EHFrameReader.readSLEB128(DataAlignmentFactor)) + if (auto Err = RecordReader.readSLEB128(DataAlignmentFactor)) return Err; if (DataAlignmentFactor != -8) return make_error<JITLinkError>("Unsupported CIE data alignment factor " + @@ -204,14 +311,14 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, } // Skip the return address register field. - if (auto Err = EHFrameReader.skip(1)) + if (auto Err = RecordReader.skip(1)) return Err; uint64_t AugmentationDataLength = 0; - if (auto Err = EHFrameReader.readULEB128(AugmentationDataLength)) + if (auto Err = RecordReader.readULEB128(AugmentationDataLength)) return Err; - uint32_t AugmentationDataStartOffset = EHFrameReader.getOffset(); + uint32_t AugmentationDataStartOffset = RecordReader.getOffset(); uint8_t *NextField = &AugInfo->Fields[0]; while (uint8_t Field = *NextField++) { @@ -219,7 +326,7 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, case 'L': { CIEInfo.FDEsHaveLSDAField = true; uint8_t LSDAPointerEncoding; - if (auto Err = EHFrameReader.readInteger(LSDAPointerEncoding)) + if (auto Err = RecordReader.readInteger(LSDAPointerEncoding)) return Err; if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) return make_error<JITLinkError>( @@ -230,7 +337,7 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, } case 'P': { uint8_t PersonalityPointerEncoding = 0; - if (auto Err = EHFrameReader.readInteger(PersonalityPointerEncoding)) + if (auto Err = RecordReader.readInteger(PersonalityPointerEncoding)) return Err; if (PersonalityPointerEncoding != (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4)) @@ -240,13 +347,13 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + formatv("{0:x16}", CIESymbol.getAddress())); uint32_t PersonalityPointerAddress; - if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) + if (auto Err = RecordReader.readInteger(PersonalityPointerAddress)) return Err; break; } case 'R': { uint8_t FDEPointerEncoding; - if (auto Err = EHFrameReader.readInteger(FDEPointerEncoding)) + if (auto Err = RecordReader.readInteger(FDEPointerEncoding)) return Err; if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) return make_error<JITLinkError>( @@ -261,107 +368,265 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, } } - if (EHFrameReader.getOffset() - AugmentationDataStartOffset > + if (RecordReader.getOffset() - AugmentationDataStartOffset > AugmentationDataLength) return make_error<JITLinkError>("Read past the end of the augmentation " "data while parsing fields"); - assert(!CIEInfos.count(CIESymbol.getAddress()) && + assert(!PC.CIEInfos.count(CIESymbol.getAddress()) && "Multiple CIEs recorded at the same address?"); - CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); + PC.CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); return Error::success(); } -Error EHFrameBinaryParser::processFDE(size_t RecordOffset, size_t RecordLength, - JITTargetAddress CIEPointerAddress, - uint32_t CIEPointer) { - LLVM_DEBUG(dbgs() << " Record is FDE\n"); +Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, + size_t RecordOffset, size_t RecordLength, + size_t CIEDeltaFieldOffset, + uint32_t CIEDelta, + BlockEdgeMap &BlockEdges) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); - LLVM_DEBUG({ - dbgs() << " CIE pointer: " - << format("0x%016" PRIx64, CIEPointerAddress - CIEPointer) << "\n"; - }); - - auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer); - if (CIEInfoItr == CIEInfos.end()) - return make_error<JITLinkError>( - "FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset) + - " points to non-existant CIE at " + - formatv("{0:x16}", CIEPointerAddress - CIEPointer)); - auto &CIEInfo = CIEInfoItr->second; + JITTargetAddress RecordAddress = B.getAddress() + RecordOffset; - // Read and sanity check the PC-start pointer and size. - JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); + auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); + BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); - auto PCBeginDelta = readAbsolutePointer(); - if (!PCBeginDelta) - return PCBeginDelta.takeError(); + // Skip past the CIE delta field: we've already read this far. + RecordReader.setOffset(CIEDeltaFieldOffset + 4); - JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; - LLVM_DEBUG({ - dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; - }); + auto &FDESymbol = + PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); - auto *TargetSymbol = getSymbolAtAddress(PCBegin); + CIEInformation *CIEInfo = nullptr; - if (!TargetSymbol) - return make_error<JITLinkError>("FDE PC-begin " + - formatv("{0:x16}", PCBegin) + - " does not point at symbol"); + { + // Process the CIE pointer field. + auto CIEEdgeItr = BlockEdges.find(RecordOffset + CIEDeltaFieldOffset); + JITTargetAddress CIEAddress = + RecordAddress + CIEDeltaFieldOffset - CIEDelta; + if (CIEEdgeItr == BlockEdges.end()) { + + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + << " to CIE at: " << formatv("{0:x16}", CIEAddress) << "\n"; + }); + if (auto CIEInfoOrErr = PC.findCIEInfo(CIEAddress)) + CIEInfo = *CIEInfoOrErr; + else + return CIEInfoOrErr.takeError(); + assert(CIEInfo->CIESymbol && "CIEInfo has no CIE symbol set"); + B.addEdge(FDEToCIE, RecordOffset + CIEDeltaFieldOffset, + *CIEInfo->CIESymbol, 0); + } else { + LLVM_DEBUG({ + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + << " to CIE at " << formatv("{0:x16}", CIEAddress) << "\n"; + }); + auto &EI = CIEEdgeItr->second; + if (EI.Addend) + return make_error<JITLinkError>( + "CIE edge at " + + formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + + " has non-zero addend"); + if (auto CIEInfoOrErr = PC.findCIEInfo(EI.Target->getAddress())) + CIEInfo = *CIEInfoOrErr; + else + return CIEInfoOrErr.takeError(); + } + } - if (TargetSymbol->getAddress() != PCBegin) - return make_error<JITLinkError>( - "FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point to start of symbol at " + - formatv("{0:x16}", TargetSymbol->getAddress())); + { + // Process the PC-Begin field. + Block *PCBeginBlock = nullptr; + JITTargetAddress PCBeginFieldOffset = RecordReader.getOffset(); + auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset); + if (PCEdgeItr == BlockEdges.end()) { + auto PCBeginDelta = readAbsolutePointer(PC.G, RecordReader); + if (!PCBeginDelta) + return PCBeginDelta.takeError(); + JITTargetAddress PCBegin = + RecordAddress + PCBeginFieldOffset + *PCBeginDelta; + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) + << " to PC at " << formatv("{0:x16}", PCBegin) << "\n"; + }); + auto PCBeginSym = getOrCreateSymbol(PC, PCBegin); + if (!PCBeginSym) + return PCBeginSym.takeError(); + B.addEdge(FDEToPCBegin, RecordOffset + PCBeginFieldOffset, *PCBeginSym, + 0); + PCBeginBlock = &PCBeginSym->getBlock(); + } else { + auto &EI = PCEdgeItr->second; + LLVM_DEBUG({ + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) + << " to PC at " << formatv("{0:x16}", EI.Target->getAddress()); + if (EI.Addend) + dbgs() << " + " << formatv("{0:x16}", EI.Addend); + dbgs() << "\n"; + }); + + // Make sure the existing edge points at a defined block. + if (!EI.Target->isDefined()) { + auto EdgeAddr = RecordAddress + PCBeginFieldOffset; + return make_error<JITLinkError>("FDE edge at " + + formatv("{0:x16}", EdgeAddr) + + " points at external block"); + } + PCBeginBlock = &EI.Target->getBlock(); + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + return Err; + } - LLVM_DEBUG(dbgs() << " FDE target: " << *TargetSymbol << "\n"); + // Add a keep-alive edge from the FDE target to the FDE to ensure that the + // FDE is kept alive if its target is. + assert(PCBeginBlock && "PC-begin block not recorded"); + PCBeginBlock->addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + } // Skip over the PC range size field. - if (auto Err = EHFrameReader.skip(PointerSize)) + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) return Err; - Symbol *LSDASymbol = nullptr; - JITTargetAddress LSDAAddress = 0; - if (CIEInfo.FDEsHaveLSDAField) { + if (CIEInfo->FDEsHaveLSDAField) { uint64_t AugmentationDataSize; - if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) + if (auto Err = RecordReader.readULEB128(AugmentationDataSize)) return Err; - if (AugmentationDataSize != PointerSize) + if (AugmentationDataSize != PC.G.getPointerSize()) return make_error<JITLinkError>( "Unexpected FDE augmentation data size (expected " + - Twine(PointerSize) + ", got " + Twine(AugmentationDataSize) + - ") for FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset)); - LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); - auto LSDADelta = readAbsolutePointer(); - if (!LSDADelta) - return LSDADelta.takeError(); + Twine(PC.G.getPointerSize()) + ", got " + + Twine(AugmentationDataSize) + ") for FDE at " + + formatv("{0:x16}", RecordAddress)); + + JITTargetAddress LSDAFieldOffset = RecordReader.getOffset(); + auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset); + if (LSDAEdgeItr == BlockEdges.end()) { + auto LSDADelta = readAbsolutePointer(PC.G, RecordReader); + if (!LSDADelta) + return LSDADelta.takeError(); + JITTargetAddress LSDA = RecordAddress + LSDAFieldOffset + *LSDADelta; + auto LSDASym = getOrCreateSymbol(PC, LSDA); + if (!LSDASym) + return LSDASym.takeError(); + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) + << " to LSDA at " << formatv("{0:x16}", LSDA) << "\n"; + }); + B.addEdge(FDEToLSDA, RecordOffset + LSDAFieldOffset, *LSDASym, 0); + } else { + LLVM_DEBUG({ + auto &EI = LSDAEdgeItr->second; + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) + << " to LSDA at " << formatv("{0:x16}", EI.Target->getAddress()); + if (EI.Addend) + dbgs() << " + " << formatv("{0:x16}", EI.Addend); + dbgs() << "\n"; + }); + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + return Err; + } + } else { + LLVM_DEBUG(dbgs() << " Record does not have LSDA field.\n"); + } - JITTargetAddress LSDA = LSDAAddress + *LSDADelta; + return Error::success(); +} - LSDASymbol = getSymbolAtAddress(LSDA); +Expected<EHFrameEdgeFixer::AugmentationInfo> +EHFrameEdgeFixer::parseAugmentationString(BinaryStreamReader &RecordReader) { + AugmentationInfo AugInfo; + uint8_t NextChar; + uint8_t *NextField = &AugInfo.Fields[0]; - if (!LSDASymbol) - return make_error<JITLinkError>("FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point at symbol"); + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); - if (LSDASymbol->getAddress() != LSDA) - return make_error<JITLinkError>( - "FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point to start of symbol at " + - formatv("{0:x16}", LSDASymbol->getAddress())); + while (NextChar != 0) { + switch (NextChar) { + case 'z': + AugInfo.AugmentationDataPresent = true; + break; + case 'e': + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); + if (NextChar != 'h') + return make_error<JITLinkError>("Unrecognized substring e" + + Twine(NextChar) + + " in augmentation string"); + AugInfo.EHDataFieldPresent = true; + break; + case 'L': + case 'P': + case 'R': + *NextField++ = NextChar; + break; + default: + return make_error<JITLinkError>("Unrecognized character " + + Twine(NextChar) + + " in augmentation string"); + } - LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDASymbol << "\n"); + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); } - JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset; - auto FDESymbol = createFDERecord( - RecordAddress, EHFrameContent.substr(RecordOffset, RecordLength), - *CIEInfo.CIESymbol, CIEPointerAddress - RecordAddress, *TargetSymbol, - PCBeginAddress - RecordAddress, LSDASymbol, LSDAAddress - RecordAddress); + return std::move(AugInfo); +} + +Expected<JITTargetAddress> +EHFrameEdgeFixer::readAbsolutePointer(LinkGraph &G, + BinaryStreamReader &RecordReader) { + static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), + "Result must be able to hold a uint64_t"); + JITTargetAddress Addr; + if (G.getPointerSize() == 8) { + if (auto Err = RecordReader.readInteger(Addr)) + return std::move(Err); + } else if (G.getPointerSize() == 4) { + uint32_t Addr32; + if (auto Err = RecordReader.readInteger(Addr32)) + return std::move(Err); + Addr = Addr32; + } else + llvm_unreachable("Pointer size is not 32-bit or 64-bit"); + return Addr; +} + +Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, + JITTargetAddress Addr) { + Symbol *CanonicalSym = nullptr; + + auto UpdateCanonicalSym = [&](Symbol *Sym) { + if (!CanonicalSym || Sym->getLinkage() < CanonicalSym->getLinkage() || + Sym->getScope() < CanonicalSym->getScope() || + (Sym->hasName() && !CanonicalSym->hasName()) || + Sym->getName() < CanonicalSym->getName()) + CanonicalSym = Sym; + }; + + if (auto *SymbolsAtAddr = PC.AddrToSyms.getSymbolsAt(Addr)) + for (auto *Sym : *SymbolsAtAddr) + UpdateCanonicalSym(Sym); + + // If we found an existing symbol at the given address then use it. + if (CanonicalSym) + return *CanonicalSym; - return FDESymbol.takeError(); + // Otherwise search for a block covering the address and create a new symbol. + auto *B = PC.AddrToBlock.getBlockCovering(Addr); + if (!B) + return make_error<JITLinkError>("No symbol or block covering address " + + formatv("{0:x16}", Addr)); + + return PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); } // Determine whether we can register EH tables. @@ -444,9 +709,6 @@ Error walkAppleEHFrameSection(const char *const SectionStart, else Size += 4; uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); - if (Offset != 0) - if (auto Err = HandleFDE(CurCFIRecord)) - return Err; LLVM_DEBUG({ dbgs() << "Registering eh-frame section:\n"; @@ -456,6 +718,11 @@ Error walkAppleEHFrameSection(const char *const SectionStart, dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); dbgs() << " ]\n"; }); + + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + CurCFIRecord += Size; Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index 6f9f68ad8382..a8cd32c664dc 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,30 +21,30 @@ namespace llvm { namespace jitlink { -/// A generic binary parser for eh-frame sections. -/// -/// Adds blocks and symbols representing CIE and FDE entries to a JITLink graph. -/// -/// This parser assumes that the user has already verified that the EH-frame's -/// address range does not overlap any other section/symbol, so that generated -/// CIE/FDE records do not overlap other sections/symbols. -class EHFrameBinaryParser { +/// A LinkGraph pass that splits blocks in an eh-frame section into sub-blocks +/// representing individual eh-frames. +/// EHFrameSplitter should not be run without EHFrameEdgeFixer, which is +/// responsible for adding FDE-to-CIE edges. +class EHFrameSplitter { public: - EHFrameBinaryParser(JITTargetAddress EHFrameAddress, StringRef EHFrameContent, - unsigned PointerSize, support::endianness Endianness); - virtual ~EHFrameBinaryParser() {} + EHFrameSplitter(StringRef EHFrameSectionName); + Error operator()(LinkGraph &G); - Error addToGraph(); +private: + Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); + + StringRef EHFrameSectionName; +}; + +/// A LinkGraph pass that adds missing FDE-to-CIE, FDE-to-PC and FDE-to-LSDA +/// edges. +class EHFrameEdgeFixer { +public: + EHFrameEdgeFixer(StringRef EHFrameSectionName, Edge::Kind FDEToCIE, + Edge::Kind FDEToPCBegin, Edge::Kind FDEToLSDA); + Error operator()(LinkGraph &G); private: - virtual void anchor(); - virtual Symbol *getSymbolAtAddress(JITTargetAddress Addr) = 0; - virtual Symbol &createCIERecord(JITTargetAddress RecordAddr, - StringRef RecordContent) = 0; - virtual Expected<Symbol &> - createFDERecord(JITTargetAddress RecordAddr, StringRef RecordContent, - Symbol &CIE, size_t CIEOffset, Symbol &Func, - size_t FuncOffset, Symbol *LSDA, size_t LSDAOffset) = 0; struct AugmentationInfo { bool AugmentationDataPresent = false; @@ -52,12 +52,6 @@ private: uint8_t Fields[4] = {0x0, 0x0, 0x0, 0x0}; }; - Expected<AugmentationInfo> parseAugmentationString(); - Expected<JITTargetAddress> readAbsolutePointer(); - Error processCIE(size_t RecordOffset, size_t RecordLength); - Error processFDE(size_t RecordOffset, size_t RecordLength, - JITTargetAddress CIEPointerOffset, uint32_t CIEPointer); - struct CIEInformation { CIEInformation() = default; CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} @@ -65,11 +59,51 @@ private: bool FDEsHaveLSDAField = false; }; - JITTargetAddress EHFrameAddress; - StringRef EHFrameContent; - unsigned PointerSize; - BinaryStreamReader EHFrameReader; - DenseMap<JITTargetAddress, CIEInformation> CIEInfos; + struct EdgeTarget { + EdgeTarget() = default; + EdgeTarget(const Edge &E) : Target(&E.getTarget()), Addend(E.getAddend()) {} + + Symbol *Target = nullptr; + Edge::AddendT Addend = 0; + }; + + using BlockEdgeMap = DenseMap<Edge::OffsetT, EdgeTarget>; + using CIEInfosMap = DenseMap<JITTargetAddress, CIEInformation>; + + struct ParseContext { + ParseContext(LinkGraph &G) : G(G) {} + + Expected<CIEInformation *> findCIEInfo(JITTargetAddress Address) { + auto I = CIEInfos.find(Address); + if (I == CIEInfos.end()) + return make_error<JITLinkError>("No CIE found at address " + + formatv("{0:x16}", Address)); + return &I->second; + } + + LinkGraph &G; + CIEInfosMap CIEInfos; + BlockAddressMap AddrToBlock; + SymbolAddressMap AddrToSyms; + }; + + Error processBlock(ParseContext &PC, Block &B); + Error processCIE(ParseContext &PC, Block &B, size_t RecordOffset, + size_t RecordLength, size_t CIEDeltaFieldOffset); + Error processFDE(ParseContext &PC, Block &B, size_t RecordOffset, + size_t RecordLength, size_t CIEDeltaFieldOffset, + uint32_t CIEDelta, BlockEdgeMap &BlockEdges); + + Expected<AugmentationInfo> + parseAugmentationString(BinaryStreamReader &RecordReader); + Expected<JITTargetAddress> + readAbsolutePointer(LinkGraph &G, BinaryStreamReader &RecordReader); + Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, JITTargetAddress Addr); + + StringRef EHFrameSectionName; + Edge::Kind FDEToCIE; + Edge::Kind FDEToPCBegin; + Edge::Kind FDEToLSDA; }; } // end namespace jitlink diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 1e19038951ac..6c924f889577 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -145,14 +145,89 @@ void printEdge(raw_ostream &OS, const Block &B, const Edge &E, Section::~Section() { for (auto *Sym : Symbols) Sym->~Symbol(); -} - -LinkGraph::~LinkGraph() { - // Destroy blocks. for (auto *B : Blocks) B->~Block(); } +Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex, + SplitBlockCache *Cache) { + + assert(SplitIndex > 0 && "splitBlock can not be called with SplitIndex == 0"); + + // If the split point covers all of B then just return B. + if (SplitIndex == B.getSize()) + return B; + + assert(SplitIndex < B.getSize() && "SplitIndex out of range"); + + // Create the new block covering [ 0, SplitIndex ). + auto &NewBlock = + B.isZeroFill() + ? createZeroFillBlock(B.getSection(), SplitIndex, B.getAddress(), + B.getAlignment(), B.getAlignmentOffset()) + : createContentBlock( + B.getSection(), B.getContent().substr(0, SplitIndex), + B.getAddress(), B.getAlignment(), B.getAlignmentOffset()); + + // Modify B to cover [ SplitIndex, B.size() ). + B.setAddress(B.getAddress() + SplitIndex); + B.setContent(B.getContent().substr(SplitIndex)); + B.setAlignmentOffset((B.getAlignmentOffset() + SplitIndex) % + B.getAlignment()); + + // Handle edge transfer/update. + { + // Copy edges to NewBlock (recording their iterators so that we can remove + // them from B), and update of Edges remaining on B. + std::vector<Block::edge_iterator> EdgesToRemove; + for (auto I = B.edges().begin(), E = B.edges().end(); I != E; ++I) { + if (I->getOffset() < SplitIndex) { + NewBlock.addEdge(*I); + EdgesToRemove.push_back(I); + } else + I->setOffset(I->getOffset() - SplitIndex); + } + + // Remove edges that were transfered to NewBlock from B. + while (!EdgesToRemove.empty()) { + B.removeEdge(EdgesToRemove.back()); + EdgesToRemove.pop_back(); + } + } + + // Handle symbol transfer/update. + { + // Initialize the symbols cache if necessary. + SplitBlockCache LocalBlockSymbolsCache; + if (!Cache) + Cache = &LocalBlockSymbolsCache; + if (*Cache == None) { + *Cache = SplitBlockCache::value_type(); + for (auto *Sym : B.getSection().symbols()) + if (&Sym->getBlock() == &B) + (*Cache)->push_back(Sym); + + llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getOffset() > RHS->getOffset(); + }); + } + auto &BlockSymbols = **Cache; + + // Transfer all symbols with offset less than SplitIndex to NewBlock. + while (!BlockSymbols.empty() && + BlockSymbols.back()->getOffset() < SplitIndex) { + BlockSymbols.back()->setBlock(NewBlock); + BlockSymbols.pop_back(); + } + + // Update offsets for all remaining symbols in B. + for (auto *Sym : BlockSymbols) + Sym->setOffset(Sym->getOffset() - SplitIndex); + } + + return NewBlock; +} + void LinkGraph::dump(raw_ostream &OS, std::function<StringRef(Edge::Kind)> EdgeKindToName) { if (!EdgeKindToName) @@ -191,6 +266,16 @@ void LinkGraph::dump(raw_ostream &OS, << "\n"; } +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) { + switch (LF) { + case SymbolLookupFlags::RequiredSymbol: + return OS << "RequiredSymbol"; + case SymbolLookupFlags::WeaklyReferencedSymbol: + return OS << "WeaklyReferencedSymbol"; + } + llvm_unreachable("Unrecognized lookup flags"); +} + void JITLinkAsyncLookupContinuation::anchor() {} JITLinkContext::~JITLinkContext() {} diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index d4270b5aa796..7b594fd2c0ea 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -151,9 +151,12 @@ JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { for (auto &KV : Layout) { auto CompareBlocks = [](const Block *LHS, const Block *RHS) { + // Sort by section, address and size if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal()) return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal(); - return LHS->getOrdinal() < RHS->getOrdinal(); + if (LHS->getAddress() != RHS->getAddress()) + return LHS->getAddress() < RHS->getAddress(); + return LHS->getSize() < RHS->getSize(); }; auto &SegLists = KV.second; @@ -254,25 +257,35 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { return Error::success(); } -DenseSet<StringRef> JITLinkerBase::getExternalSymbolNames() const { +JITLinkContext::LookupMap JITLinkerBase::getExternalSymbolNames() const { // Identify unresolved external symbols. - DenseSet<StringRef> UnresolvedExternals; + JITLinkContext::LookupMap UnresolvedExternals; for (auto *Sym : G->external_symbols()) { assert(Sym->getAddress() == 0 && "External has already been assigned an address"); assert(Sym->getName() != StringRef() && Sym->getName() != "" && "Externals must be named"); - UnresolvedExternals.insert(Sym->getName()); + SymbolLookupFlags LookupFlags = + Sym->getLinkage() == Linkage::Weak + ? SymbolLookupFlags::WeaklyReferencedSymbol + : SymbolLookupFlags::RequiredSymbol; + UnresolvedExternals[Sym->getName()] = LookupFlags; } return UnresolvedExternals; } void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { for (auto *Sym : G->external_symbols()) { + assert(Sym->getOffset() == 0 && + "External symbol is not at the start of its addressable block"); assert(Sym->getAddress() == 0 && "Symbol already resolved"); assert(!Sym->isDefined() && "Symbol being resolved is already defined"); - assert(Result.count(Sym->getName()) && "Missing resolution for symbol"); - Sym->getAddressable().setAddress(Result[Sym->getName()].getAddress()); + auto ResultI = Result.find(Sym->getName()); + if (ResultI != Result.end()) + Sym->getAddressable().setAddress(ResultI->second.getAddress()); + else + assert(Sym->getLinkage() == Linkage::Weak && + "Failed to resolve non-weak reference"); } LLVM_DEBUG({ @@ -282,8 +295,11 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { << formatv("{0:x16}", Sym->getAddress()) << "\n"; }); assert(llvm::all_of(G->external_symbols(), - [](Symbol *Sym) { return Sym->getAddress() != 0; }) && - "All symbols should have been resolved by this point"); + [](Symbol *Sym) { + return Sym->getAddress() != 0 || + Sym->getLinkage() == Linkage::Weak; + }) && + "All strong external symbols should have been resolved by now"); } void JITLinkerBase::deallocateAndBailOut(Error Err) { diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index 07dee6cee200..d5687b7afc96 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -106,7 +106,7 @@ private: SegmentLayoutMap layOutBlocks(); Error allocateSegments(const SegmentLayoutMap &Layout); - DenseSet<StringRef> getExternalSymbolNames() const; + JITLinkContext::LookupMap getExternalSymbolNames() const; void applyLookupResult(AsyncLookupResult LR); void deallocateAndBailOut(Error Err); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index 7366f53ebf36..701f108a9a21 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -179,7 +179,9 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { llvm::sort(Sections, [](const NormalizedSection *LHS, const NormalizedSection *RHS) { assert(LHS && RHS && "Null section?"); - return LHS->Address < RHS->Address; + if (LHS->Address != RHS->Address) + return LHS->Address < RHS->Address; + return LHS->Size < RHS->Size; }); for (unsigned I = 0, E = Sections.size() - 1; I != E; ++I) { @@ -311,7 +313,7 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { return make_error<JITLinkError>("Anonymous common symbol at index " + Twine(KV.first)); NSym.GraphSymbol = &G->addCommonSymbol( - *NSym.Name, NSym.S, getCommonSection(), NSym.Value, 0, + *NSym.Name, NSym.S, getCommonSection(), 0, NSym.Value, 1ull << MachO::GET_COMM_ALIGN(NSym.Desc), NSym.Desc & MachO::N_NO_DEAD_STRIP); } else { @@ -319,7 +321,9 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { return make_error<JITLinkError>("Anonymous external symbol at " "index " + Twine(KV.first)); - NSym.GraphSymbol = &G->addExternalSymbol(*NSym.Name, 0); + NSym.GraphSymbol = &G->addExternalSymbol( + *NSym.Name, 0, + NSym.Desc & MachO::N_WEAK_REF ? Linkage::Weak : Linkage::Strong); } break; case MachO::N_ABS: diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index e1123cd11048..91b1d5a22387 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -30,74 +30,6 @@ public: Expected<std::unique_ptr<LinkGraph>> buildGraph(); protected: - class MachOEHFrameBinaryParser : public EHFrameBinaryParser { - public: - MachOEHFrameBinaryParser(MachOLinkGraphBuilder &Builder, - JITTargetAddress EHFrameAddress, - StringRef EHFrameContent, Section &EHFrameSection, - uint64_t CIEAlignment, uint64_t FDEAlignment, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) - : EHFrameBinaryParser(EHFrameAddress, EHFrameContent, - Builder.getGraph().getPointerSize(), - Builder.getGraph().getEndianness()), - Builder(Builder), EHFrameSection(EHFrameSection), - CIEAlignment(CIEAlignment), FDEAlignment(FDEAlignment), - FDEToCIERelocKind(FDEToCIERelocKind), - FDEToTargetRelocKind(FDEToTargetRelocKind) {} - - Symbol *getSymbolAtAddress(JITTargetAddress Address) override { - if (auto *Sym = Builder.getSymbolByAddress(Address)) - if (Sym->getAddress() == Address) - return Sym; - return nullptr; - } - - Symbol &createCIERecord(JITTargetAddress RecordAddr, - StringRef RecordContent) override { - auto &G = Builder.getGraph(); - auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, - CIEAlignment, 0); - auto &CIESymbol = - G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); - Builder.setCanonicalSymbol(CIESymbol); - return CIESymbol; - } - - Expected<Symbol &> createFDERecord(JITTargetAddress RecordAddr, - StringRef RecordContent, Symbol &CIE, - size_t CIEOffset, Symbol &Func, - size_t FuncOffset, Symbol *LSDA, - size_t LSDAOffset) override { - auto &G = Builder.getGraph(); - auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, - FDEAlignment, 0); - - // Add edges to CIE, Func, and (conditionally) LSDA. - B.addEdge(FDEToCIERelocKind, CIEOffset, CIE, 0); - B.addEdge(FDEToTargetRelocKind, FuncOffset, Func, 0); - - if (LSDA) - B.addEdge(FDEToTargetRelocKind, LSDAOffset, *LSDA, 0); - - auto &FDESymbol = - G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); - - // Add a keep-alive relocation from the function to the FDE to ensure it - // is not dead stripped. - Func.getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); - - return FDESymbol; - } - - private: - MachOLinkGraphBuilder &Builder; - Section &EHFrameSection; - uint64_t CIEAlignment; - uint64_t FDEAlignment; - Edge::Kind FDEToCIERelocKind; - Edge::Kind FDEToTargetRelocKind; - }; struct NormalizedSymbol { friend class MachOLinkGraphBuilder; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 945343bff89d..944767449ce2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -27,19 +27,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) : MachOLinkGraphBuilder(Obj), - NumSymbols(Obj.getSymtabLoadCommand().nsyms) { - addCustomSectionParser( - "__eh_frame", [this](NormalizedSection &EHFrameSection) { - if (!EHFrameSection.Data) - return make_error<JITLinkError>( - "__eh_frame section is marked zero-fill"); - return MachOEHFrameBinaryParser( - *this, EHFrameSection.Address, - StringRef(EHFrameSection.Data, EHFrameSection.Size), - *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) - .addToGraph(); - }); - } + NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: static Expected<MachOARM64RelocationKind> diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index d83787ffd598..69ec72aae292 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -26,19 +26,7 @@ namespace { class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj) { - addCustomSectionParser( - "__eh_frame", [this](NormalizedSection &EHFrameSection) { - if (!EHFrameSection.Data) - return make_error<JITLinkError>( - "__eh_frame section is marked zero-fill"); - return MachOEHFrameBinaryParser( - *this, EHFrameSection.Address, - StringRef(EHFrameSection.Data, EHFrameSection.Size), - *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) - .addToGraph(); - }); - } + : MachOLinkGraphBuilder(Obj) {} private: static Expected<MachOX86RelocationKind> @@ -264,7 +252,7 @@ private: TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); - Addend = *(const ulittle32_t *)FixupContent; + Addend = *(const little32_t *)FixupContent; break; case Pointer32: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) @@ -296,12 +284,12 @@ private: TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); - Addend = *(const ulittle32_t *)FixupContent + + Addend = *(const little32_t *)FixupContent + (1 << (*Kind - PCRel32Minus1)); break; case PCRel32Anon: { JITTargetAddress TargetAddress = - FixupAddress + 4 + *(const ulittle32_t *)FixupContent; + FixupAddress + 4 + *(const little32_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else @@ -315,7 +303,7 @@ private: JITTargetAddress Delta = static_cast<JITTargetAddress>(1ULL << (*Kind - PCRel32Minus1Anon)); JITTargetAddress TargetAddress = - FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; + FixupAddress + 4 + Delta + *(const little32_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else @@ -566,6 +554,11 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { Triple TT("x86_64-apple-macosx"); if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add eh-frame passses. + Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame")); + Config.PrePrunePasses.push_back( + EHFrameEdgeFixer("__eh_frame", NegDelta32, Delta64, Delta64)); + // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); diff --git a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 75ddbc30445d..f26835ff8a08 100644 --- a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -162,7 +162,8 @@ void CompileOnDemandLayer::emit(MaterializationResponsibility R, return; } - R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), true)); + R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), + JITDylibLookupFlags::MatchAllSymbols)); R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), std::move(Callables), AliaseeImpls)); } @@ -171,18 +172,22 @@ CompileOnDemandLayer::PerDylibResources & CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) { auto I = DylibResources.find(&TargetD); if (I == DylibResources.end()) { - auto &ImplD = getExecutionSession().createJITDylib( - TargetD.getName() + ".impl", false); - TargetD.withSearchOrderDo([&](const JITDylibSearchList &TargetSearchOrder) { - auto NewSearchOrder = TargetSearchOrder; - assert(!NewSearchOrder.empty() && - NewSearchOrder.front().first == &TargetD && - NewSearchOrder.front().second == true && - "TargetD must be at the front of its own search order and match " - "non-exported symbol"); - NewSearchOrder.insert(std::next(NewSearchOrder.begin()), {&ImplD, true}); - ImplD.setSearchOrder(std::move(NewSearchOrder), false); - }); + auto &ImplD = + getExecutionSession().createJITDylib(TargetD.getName() + ".impl"); + TargetD.withSearchOrderDo( + [&](const JITDylibSearchOrder &TargetSearchOrder) { + auto NewSearchOrder = TargetSearchOrder; + assert( + !NewSearchOrder.empty() && + NewSearchOrder.front().first == &TargetD && + NewSearchOrder.front().second == + JITDylibLookupFlags::MatchAllSymbols && + "TargetD must be at the front of its own search order and match " + "non-exported symbol"); + NewSearchOrder.insert(std::next(NewSearchOrder.begin()), + {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); + ImplD.setSearchOrder(std::move(NewSearchOrder), false); + }); PerDylibResources PDR(ImplD, BuildIndirectStubsManager()); I = DylibResources.insert(std::make_pair(&TargetD, std::move(PDR))).first; } diff --git a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp index f8251627a4ef..f5671d90420a 100644 --- a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -43,8 +43,7 @@ SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { } auto ObjBuffer = std::make_unique<SmallVectorMemoryBuffer>( - std::move(ObjBufferSV), - "<in memory object compiled from " + M.getModuleIdentifier() + ">"); + std::move(ObjBufferSV), M.getModuleIdentifier() + "-jitted-objectbuffer"); auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index 5c7d888c2d6e..63ef889dae46 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -7,6 +7,8 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Core.h" + +#include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/IR/Mangler.h" @@ -77,16 +79,19 @@ bool flagsMatchCLOpts(const JITSymbolFlags &Flags) { #endif // NDEBUG } -// Prints a set of items, filtered by an user-supplied predicate. -template <typename Set, typename Pred = PrintAll<typename Set::value_type>> -class SetPrinter { +// Prints a sequence of items, filtered by an user-supplied predicate. +template <typename Sequence, + typename Pred = PrintAll<typename Sequence::value_type>> +class SequencePrinter { public: - SetPrinter(const Set &S, Pred ShouldPrint = Pred()) - : S(S), ShouldPrint(std::move(ShouldPrint)) {} + SequencePrinter(const Sequence &S, char OpenSeq, char CloseSeq, + Pred ShouldPrint = Pred()) + : S(S), OpenSeq(OpenSeq), CloseSeq(CloseSeq), + ShouldPrint(std::move(ShouldPrint)) {} void printTo(llvm::raw_ostream &OS) const { bool PrintComma = false; - OS << "{"; + OS << OpenSeq; for (auto &E : S) { if (ShouldPrint(E)) { if (PrintComma) @@ -95,23 +100,26 @@ public: PrintComma = true; } } - OS << " }"; + OS << ' ' << CloseSeq; } private: - const Set &S; + const Sequence &S; + char OpenSeq; + char CloseSeq; mutable Pred ShouldPrint; }; -template <typename Set, typename Pred> -SetPrinter<Set, Pred> printSet(const Set &S, Pred P = Pred()) { - return SetPrinter<Set, Pred>(S, std::move(P)); +template <typename Sequence, typename Pred> +SequencePrinter<Sequence, Pred> printSequence(const Sequence &S, char OpenSeq, + char CloseSeq, Pred P = Pred()) { + return SequencePrinter<Sequence, Pred>(S, OpenSeq, CloseSeq, std::move(P)); } -// Render a SetPrinter by delegating to its printTo method. -template <typename Set, typename Pred> +// Render a SequencePrinter by delegating to its printTo method. +template <typename Sequence, typename Pred> llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, - const SetPrinter<Set, Pred> &Printer) { + const SequencePrinter<Sequence, Pred> &Printer) { Printer.printTo(OS); return OS; } @@ -147,7 +155,11 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym) { } raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { - return OS << printSet(Symbols, PrintAll<SymbolStringPtr>()); + return OS << printSequence(Symbols, '{', '}', PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols) { + return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>()); } raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { @@ -182,11 +194,13 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { } raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { - return OS << printSet(SymbolFlags, PrintSymbolFlagsMapElemsMatchingCLOpts()); + return OS << printSequence(SymbolFlags, '{', '}', + PrintSymbolFlagsMapElemsMatchingCLOpts()); } raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { - return OS << printSet(Symbols, PrintSymbolMapElemsMatchingCLOpts()); + return OS << printSequence(Symbols, '{', '}', + PrintSymbolMapElemsMatchingCLOpts()); } raw_ostream &operator<<(raw_ostream &OS, @@ -195,7 +209,8 @@ raw_ostream &operator<<(raw_ostream &OS, } raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { - return OS << printSet(Deps, PrintAll<SymbolDependenceMap::value_type>()); + return OS << printSequence(Deps, '{', '}', + PrintAll<SymbolDependenceMap::value_type>()); } raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU) { @@ -205,16 +220,59 @@ raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU) { return OS << ")"; } -raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs) { +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K) { + switch (K) { + case LookupKind::Static: + return OS << "Static"; + case LookupKind::DLSym: + return OS << "DLSym"; + } + llvm_unreachable("Invalid lookup kind"); +} + +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibLookupFlags &JDLookupFlags) { + switch (JDLookupFlags) { + case JITDylibLookupFlags::MatchExportedSymbolsOnly: + return OS << "MatchExportedSymbolsOnly"; + case JITDylibLookupFlags::MatchAllSymbols: + return OS << "MatchAllSymbols"; + } + llvm_unreachable("Invalid JITDylib lookup flags"); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags) { + switch (LookupFlags) { + case SymbolLookupFlags::RequiredSymbol: + return OS << "RequiredSymbol"; + case SymbolLookupFlags::WeaklyReferencedSymbol: + return OS << "WeaklyReferencedSymbol"; + } + llvm_unreachable("Invalid symbol lookup flags"); +} + +raw_ostream &operator<<(raw_ostream &OS, + const SymbolLookupSet::value_type &KV) { + return OS << "(" << KV.first << ", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet) { + return OS << printSequence(LookupSet, '{', '}', + PrintAll<SymbolLookupSet::value_type>()); +} + +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibSearchOrder &SearchOrder) { OS << "["; - if (!JDs.empty()) { - assert(JDs.front().first && "JITDylibList entries must not be null"); - OS << " (\"" << JDs.front().first->getName() << "\", " - << (JDs.front().second ? "true" : "false") << ")"; - for (auto &KV : make_range(std::next(JDs.begin()), JDs.end())) { + if (!SearchOrder.empty()) { + assert(SearchOrder.front().first && + "JITDylibList entries must not be null"); + OS << " (\"" << SearchOrder.front().first->getName() << "\", " + << SearchOrder.begin()->second << ")"; + for (auto &KV : + make_range(std::next(SearchOrder.begin(), 1), SearchOrder.end())) { assert(KV.first && "JITDylibList entries must not be null"); - OS << ", (\"" << KV.first->getName() << "\", " - << (KV.second ? "true" : "false") << ")"; + OS << ", (\"" << KV.first->getName() << "\", " << KV.second << ")"; } } OS << " ]"; @@ -262,7 +320,13 @@ void FailedToMaterialize::log(raw_ostream &OS) const { OS << "Failed to materialize symbols: " << *Symbols; } -SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) +SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) { + for (auto &Sym : Symbols) + this->Symbols.push_back(Sym); + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +SymbolsNotFound::SymbolsNotFound(SymbolNameVector Symbols) : Symbols(std::move(Symbols)) { assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); } @@ -289,7 +353,7 @@ void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { } AsynchronousSymbolQuery::AsynchronousSymbolQuery( - const SymbolNameSet &Symbols, SymbolState RequiredState, + const SymbolLookupSet &Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete) : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) { assert(RequiredState >= SymbolState::Resolved && @@ -298,8 +362,8 @@ AsynchronousSymbolQuery::AsynchronousSymbolQuery( OutstandingSymbolsCount = Symbols.size(); - for (auto &S : Symbols) - ResolvedSymbols[S] = nullptr; + for (auto &KV : Symbols) + ResolvedSymbols[KV.first] = nullptr; } void AsynchronousSymbolQuery::notifySymbolMetRequiredState( @@ -511,10 +575,10 @@ AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { } ReExportsMaterializationUnit::ReExportsMaterializationUnit( - JITDylib *SourceJD, bool MatchNonExported, SymbolAliasMap Aliases, - VModuleKey K) + JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, + SymbolAliasMap Aliases, VModuleKey K) : MaterializationUnit(extractFlags(Aliases), std::move(K)), - SourceJD(SourceJD), MatchNonExported(MatchNonExported), + SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {} StringRef ReExportsMaterializationUnit::getName() const { @@ -551,7 +615,7 @@ void ReExportsMaterializationUnit::materialize( if (!Aliases.empty()) { if (SourceJD) - R.replace(reexports(*SourceJD, std::move(Aliases), MatchNonExported)); + R.replace(reexports(*SourceJD, std::move(Aliases), SourceJDLookupFlags)); else R.replace(symbolAliases(std::move(Aliases))); } @@ -572,11 +636,11 @@ void ReExportsMaterializationUnit::materialize( // be waitin on a symbol that it itself had to resolve. Usually this will just // involve one round and a single query. - std::vector<std::pair<SymbolNameSet, std::shared_ptr<OnResolveInfo>>> + std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>> QueryInfos; while (!RequestedAliases.empty()) { SymbolNameSet ResponsibilitySymbols; - SymbolNameSet QuerySymbols; + SymbolLookupSet QuerySymbols; SymbolAliasMap QueryAliases; // Collect as many aliases as we can without including a chain. @@ -587,7 +651,7 @@ void ReExportsMaterializationUnit::materialize( continue; ResponsibilitySymbols.insert(KV.first); - QuerySymbols.insert(KV.second.Aliasee); + QuerySymbols.add(KV.second.Aliasee); QueryAliases[KV.first] = std::move(KV.second); } @@ -657,8 +721,9 @@ void ReExportsMaterializationUnit::materialize( } }; - ES.lookup(JITDylibSearchList({{&SrcJD, MatchNonExported}}), QuerySymbols, - SymbolState::Resolved, std::move(OnComplete), + ES.lookup(LookupKind::Static, + JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}), + QuerySymbols, SymbolState::Resolved, std::move(OnComplete), std::move(RegisterDependencies)); } } @@ -681,16 +746,16 @@ ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { - auto Flags = SourceJD.lookupFlags(Symbols); + SymbolLookupSet LookupSet(Symbols); + auto Flags = SourceJD.lookupFlags( + LookupKind::Static, JITDylibLookupFlags::MatchAllSymbols, LookupSet); if (!Flags) return Flags.takeError(); - if (Flags->size() != Symbols.size()) { - SymbolNameSet Unresolved = Symbols; - for (auto &KV : *Flags) - Unresolved.erase(KV.first); - return make_error<SymbolsNotFound>(std::move(Unresolved)); + if (!LookupSet.empty()) { + LookupSet.sortByName(); + return make_error<SymbolsNotFound>(LookupSet.getSymbolNames()); } SymbolAliasMap Result; @@ -703,32 +768,32 @@ buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { } ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, - bool MatchNonExported, + JITDylibLookupFlags SourceJDLookupFlags, SymbolPredicate Allow) - : SourceJD(SourceJD), MatchNonExported(MatchNonExported), + : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), Allow(std::move(Allow)) {} -Expected<SymbolNameSet> -ReexportsGenerator::tryToGenerate(JITDylib &JD, const SymbolNameSet &Names) { - orc::SymbolNameSet Added; - orc::SymbolAliasMap AliasMap; - - auto Flags = SourceJD.lookupFlags(Names); +Error ReexportsGenerator::tryToGenerate(LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) { + assert(&JD != &SourceJD && "Cannot re-export from the same dylib"); + // Use lookupFlags to find the subset of symbols that match our lookup. + auto Flags = SourceJD.lookupFlags(K, JDLookupFlags, LookupSet); if (!Flags) return Flags.takeError(); - for (auto &KV : *Flags) { - if (Allow && !Allow(KV.first)) - continue; - AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); - Added.insert(KV.first); - } + // Create an alias map. + orc::SymbolAliasMap AliasMap; + for (auto &KV : *Flags) + if (!Allow || Allow(KV.first)) + AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); - if (!Added.empty()) - cantFail(JD.define(reexports(SourceJD, AliasMap, MatchNonExported))); + if (AliasMap.empty()) + return Error::success(); - return Added; + // Define the re-exports. + return JD.define(reexports(SourceJD, AliasMap, SourceJDLookupFlags)); } JITDylib::DefinitionGenerator::~DefinitionGenerator() {} @@ -1228,11 +1293,14 @@ void JITDylib::notifyFailed(FailedSymbolsWorklist Worklist) { MI.UnemittedDependencies.clear(); // Collect queries to be failed for this MII. + AsynchronousSymbolQueryList ToDetach; for (auto &Q : MII->second.pendingQueries()) { // Add the query to the list to be failed and detach it. FailedQueries.insert(Q); - Q->detach(); + ToDetach.push_back(Q); } + for (auto &Q : ToDetach) + Q->detach(); assert(MI.Dependants.empty() && "Can not delete MaterializingInfo with dependants still attached"); @@ -1249,41 +1317,41 @@ void JITDylib::notifyFailed(FailedSymbolsWorklist Worklist) { Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbolsMap)); } -void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, - bool SearchThisJITDylibFirst, - bool MatchNonExportedInThisDylib) { - if (SearchThisJITDylibFirst) { - if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) - NewSearchOrder.insert(NewSearchOrder.begin(), - {this, MatchNonExportedInThisDylib}); - } - - ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); -} - -void JITDylib::addToSearchOrder(JITDylib &JD, bool MatchNonExported) { +void JITDylib::setSearchOrder(JITDylibSearchOrder NewSearchOrder, + bool SearchThisJITDylibFirst) { ES.runSessionLocked([&]() { - SearchOrder.push_back({&JD, MatchNonExported}); + if (SearchThisJITDylibFirst) { + SearchOrder.clear(); + if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) + SearchOrder.push_back( + std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols)); + SearchOrder.insert(SearchOrder.end(), NewSearchOrder.begin(), + NewSearchOrder.end()); + } else + SearchOrder = std::move(NewSearchOrder); }); } +void JITDylib::addToSearchOrder(JITDylib &JD, + JITDylibLookupFlags JDLookupFlags) { + ES.runSessionLocked([&]() { SearchOrder.push_back({&JD, JDLookupFlags}); }); +} + void JITDylib::replaceInSearchOrder(JITDylib &OldJD, JITDylib &NewJD, - bool MatchNonExported) { + JITDylibLookupFlags JDLookupFlags) { ES.runSessionLocked([&]() { - auto I = std::find_if(SearchOrder.begin(), SearchOrder.end(), - [&](const JITDylibSearchList::value_type &KV) { - return KV.first == &OldJD; - }); - - if (I != SearchOrder.end()) - *I = {&NewJD, MatchNonExported}; + for (auto &KV : SearchOrder) + if (KV.first == &OldJD) { + KV = {&NewJD, JDLookupFlags}; + break; + } }); } void JITDylib::removeFromSearchOrder(JITDylib &JD) { ES.runSessionLocked([&]() { auto I = std::find_if(SearchOrder.begin(), SearchOrder.end(), - [&](const JITDylibSearchList::value_type &KV) { + [&](const JITDylibSearchOrder::value_type &KV) { return KV.first == &JD; }); if (I != SearchOrder.end()) @@ -1346,63 +1414,54 @@ Error JITDylib::remove(const SymbolNameSet &Names) { }); } -Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) { +Expected<SymbolFlagsMap> +JITDylib::lookupFlags(LookupKind K, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet LookupSet) { return ES.runSessionLocked([&, this]() -> Expected<SymbolFlagsMap> { SymbolFlagsMap Result; - auto Unresolved = lookupFlagsImpl(Result, Names); - if (!Unresolved) - return Unresolved.takeError(); + lookupFlagsImpl(Result, K, JDLookupFlags, LookupSet); - /// Run any definition generators. + // Run any definition generators. for (auto &DG : DefGenerators) { - // Bail out early if we've resolved everything. - if (Unresolved->empty()) + // Bail out early if we found everything. + if (LookupSet.empty()) break; // Run this generator. - auto NewDefs = DG->tryToGenerate(*this, *Unresolved); - if (!NewDefs) - return NewDefs.takeError(); - - if (!NewDefs->empty()) { - auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs); - if (!Unresolved2) - return Unresolved2.takeError(); - (void)Unresolved2; - assert(Unresolved2->empty() && - "All fallback defs should have been found by lookupFlagsImpl"); - } + if (auto Err = DG->tryToGenerate(K, *this, JDLookupFlags, LookupSet)) + return std::move(Err); - for (auto &Name : *NewDefs) - Unresolved->erase(Name); + // Re-try the search. + lookupFlagsImpl(Result, K, JDLookupFlags, LookupSet); } + return Result; }); } -Expected<SymbolNameSet> JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, - const SymbolNameSet &Names) { - SymbolNameSet Unresolved; +void JITDylib::lookupFlagsImpl(SymbolFlagsMap &Result, LookupKind K, + JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &LookupSet) { - for (auto &Name : Names) { - auto I = Symbols.find(Name); - if (I != Symbols.end()) { - assert(!Flags.count(Name) && "Symbol already present in Flags map"); - Flags[Name] = I->second.getFlags(); - } else - Unresolved.insert(Name); - } - - return Unresolved; + LookupSet.forEachWithRemoval( + [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) -> bool { + auto I = Symbols.find(Name); + if (I == Symbols.end()) + return false; + assert(!Result.count(Name) && "Symbol already present in Flags map"); + Result[Name] = I->second.getFlags(); + return true; + }); } -Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs) { +Error JITDylib::lodgeQuery(MaterializationUnitList &MUs, + std::shared_ptr<AsynchronousSymbolQuery> &Q, + LookupKind K, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Unresolved) { assert(Q && "Query can not be null"); - if (auto Err = lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs)) + if (auto Err = lodgeQueryImpl(MUs, Q, K, JDLookupFlags, Unresolved)) return Err; // Run any definition generators. @@ -1413,104 +1472,86 @@ Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, break; // Run the generator. - auto NewDefs = DG->tryToGenerate(*this, Unresolved); - - // If the generator returns an error then bail out. - if (!NewDefs) - return NewDefs.takeError(); - - // If the generator was able to generate new definitions for any of the - // unresolved symbols then lodge the query against them. - if (!NewDefs->empty()) { - for (auto &D : *NewDefs) - Unresolved.erase(D); - - // Lodge query. This can not fail as any new definitions were added - // by the generator under the session locked. Since they can't have - // started materializing yet the can not have failed. - cantFail(lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs)); + if (auto Err = DG->tryToGenerate(K, *this, JDLookupFlags, Unresolved)) + return Err; - assert(NewDefs->empty() && - "All fallback defs should have been found by lookupImpl"); - } + // Lodge query. This can not fail as any new definitions were added + // by the generator under the session locked. Since they can't have + // started materializing yet they can not have failed. + cantFail(lodgeQueryImpl(MUs, Q, K, JDLookupFlags, Unresolved)); } return Error::success(); } -Error JITDylib::lodgeQueryImpl( - std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved, - bool MatchNonExported, - std::vector<std::unique_ptr<MaterializationUnit>> &MUs) { - - std::vector<SymbolStringPtr> ToRemove; - for (auto Name : Unresolved) { - - // Search for the name in Symbols. Skip it if not found. - auto SymI = Symbols.find(Name); - if (SymI == Symbols.end()) - continue; - - // If this is a non exported symbol and we're skipping those then skip it. - if (!SymI->second.getFlags().isExported() && !MatchNonExported) - continue; - - // If we matched against Name in JD, mark it to be removed from the - // Unresolved set. - ToRemove.push_back(Name); - - // If we matched against this symbol but it is in the error state then - // bail out and treat it as a failure to materialize. - if (SymI->second.getFlags().hasError()) { - auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); - (*FailedSymbolsMap)[this] = {Name}; - return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); - } - - // If this symbol already meets the required state for then notify the - // query and continue. - if (SymI->second.getState() >= Q->getRequiredState()) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - continue; - } - - // Otherwise this symbol does not yet meet the required state. Check whether - // it has a materializer attached, and if so prepare to run it. - if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && - "Symbol not resolved but already has address?"); - auto UMII = UnmaterializedInfos.find(Name); - assert(UMII != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - auto MU = std::move(UMII->second->MU); - assert(MU != nullptr && "Materializer should not be null"); - - // Move all symbols associated with this MaterializationUnit into - // materializing state. - for (auto &KV : MU->getSymbols()) { - auto SymK = Symbols.find(KV.first); - SymK->second.setMaterializerAttached(false); - SymK->second.setState(SymbolState::Materializing); - UnmaterializedInfos.erase(KV.first); - } +Error JITDylib::lodgeQueryImpl(MaterializationUnitList &MUs, + std::shared_ptr<AsynchronousSymbolQuery> &Q, + LookupKind K, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Unresolved) { + + return Unresolved.forEachWithRemoval( + [&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) -> Expected<bool> { + // Search for name in symbols. If not found then continue without + // removal. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + return false; + + // If this is a non exported symbol and we're matching exported symbols + // only then skip this symbol without removal. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) + return false; + + // If we matched against this symbol but it is in the error state then + // bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[this] = {Name}; + return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); + } - // Add MU to the list of MaterializationUnits to be materialized. - MUs.push_back(std::move(MU)); - } + // If this symbol already meets the required state for then notify the + // query, then remove the symbol and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + return true; + } - // Add the query to the PendingQueries list. - assert(SymI->second.isInMaterializationPhase() && - "By this line the symbol should be materializing"); - auto &MI = MaterializingInfos[Name]; - MI.addQuery(Q); - Q->addQueryDependence(*this, Name); - } + // Otherwise this symbol does not yet meet the required state. Check + // whether it has a materializer attached, and if so prepare to run it. + if (SymI->second.hasMaterializerAttached()) { + assert(SymI->second.getAddress() == 0 && + "Symbol not resolved but already has address?"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Move all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); + UnmaterializedInfos.erase(KV.first); + } - // Remove any symbols that we found. - for (auto &Name : ToRemove) - Unresolved.erase(Name); + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } - return Error::success(); + // Add the query to the PendingQueries list and continue, deleting the + // element. + assert(SymI->second.isInMaterializationPhase() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.addQuery(Q); + Q->addQueryDependence(*this, Name); + return true; + }); } Expected<SymbolNameSet> @@ -1523,7 +1564,7 @@ JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, bool QueryComplete = false; std::vector<std::unique_ptr<MaterializationUnit>> MUs; - SymbolNameSet Unresolved = std::move(Names); + SymbolLookupSet Unresolved(Names); auto Err = ES.runSessionLocked([&, this]() -> Error { QueryComplete = lookupImpl(Q, MUs, Unresolved); @@ -1535,16 +1576,13 @@ JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, break; assert(!QueryComplete && "query complete but unresolved symbols remain?"); - auto NewDefs = DG->tryToGenerate(*this, Unresolved); - if (!NewDefs) - return NewDefs.takeError(); - if (!NewDefs->empty()) { - for (auto &D : *NewDefs) - Unresolved.erase(D); - QueryComplete = lookupImpl(Q, MUs, *NewDefs); - assert(NewDefs->empty() && - "All fallback defs should have been found by lookupImpl"); - } + if (auto Err = DG->tryToGenerate(LookupKind::Static, *this, + JITDylibLookupFlags::MatchAllSymbols, + Unresolved)) + return Err; + + if (!Unresolved.empty()) + QueryComplete = lookupImpl(Q, MUs, Unresolved); } return Error::success(); }); @@ -1572,68 +1610,68 @@ JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, // for (auto &MU : MUs) // ES.dispatchMaterialization(*this, std::move(MU)); - return Unresolved; + SymbolNameSet RemainingSymbols; + for (auto &KV : Unresolved) + RemainingSymbols.insert(KV.first); + + return RemainingSymbols; } bool JITDylib::lookupImpl( std::shared_ptr<AsynchronousSymbolQuery> &Q, std::vector<std::unique_ptr<MaterializationUnit>> &MUs, - SymbolNameSet &Unresolved) { + SymbolLookupSet &Unresolved) { bool QueryComplete = false; std::vector<SymbolStringPtr> ToRemove; - for (auto Name : Unresolved) { - - // Search for the name in Symbols. Skip it if not found. - auto SymI = Symbols.find(Name); - if (SymI == Symbols.end()) - continue; - - // If we found Name, mark it to be removed from the Unresolved set. - ToRemove.push_back(Name); - - if (SymI->second.getState() >= Q->getRequiredState()) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - if (Q->isComplete()) - QueryComplete = true; - continue; - } - - // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && - "Lazy symbol should not have a resolved address"); - auto UMII = UnmaterializedInfos.find(Name); - assert(UMII != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - auto MU = std::move(UMII->second->MU); - assert(MU != nullptr && "Materializer should not be null"); - - // Kick all symbols associated with this MaterializationUnit into - // materializing state. - for (auto &KV : MU->getSymbols()) { - auto SymK = Symbols.find(KV.first); - assert(SymK != Symbols.end() && "Missing symbol table entry"); - SymK->second.setState(SymbolState::Materializing); - SymK->second.setMaterializerAttached(false); - UnmaterializedInfos.erase(KV.first); - } + Unresolved.forEachWithRemoval( + [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) -> bool { + // Search for the name in Symbols. Skip without removing if not found. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + return false; + + // If the symbol is already in the required state then notify the query + // and remove. + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + QueryComplete = true; + return true; + } - // Add MU to the list of MaterializationUnits to be materialized. - MUs.push_back(std::move(MU)); - } + // If the symbol is lazy, get the MaterialiaztionUnit for it. + if (SymI->second.hasMaterializerAttached()) { + assert(SymI->second.getAddress() == 0 && + "Lazy symbol should not have a resolved address"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Kick all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + assert(SymK != Symbols.end() && "Missing symbol table entry"); + SymK->second.setState(SymbolState::Materializing); + SymK->second.setMaterializerAttached(false); + UnmaterializedInfos.erase(KV.first); + } - // Add the query to the PendingQueries list. - assert(SymI->second.isInMaterializationPhase() && - "By this line the symbol should be materializing"); - auto &MI = MaterializingInfos[Name]; - MI.addQuery(Q); - Q->addQueryDependence(*this, Name); - } + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } - // Remove any marked symbols from the Unresolved set. - for (auto &Name : ToRemove) - Unresolved.erase(Name); + // Add the query to the PendingQueries list. + assert(SymI->second.isInMaterializationPhase() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.addQuery(Q); + Q->addQueryDependence(*this, Name); + return true; + }); return QueryComplete; } @@ -1642,11 +1680,7 @@ void JITDylib::dump(raw_ostream &OS) { ES.runSessionLocked([&, this]() { OS << "JITDylib \"" << JITDylibName << "\" (ES: " << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES)) << "):\n" - << "Search order: ["; - for (auto &KV : SearchOrder) - OS << " (\"" << KV.first->getName() << "\", " - << (KV.second ? "all" : "exported only") << ")"; - OS << " ]\n" + << "Search order: " << SearchOrder << "\n" << "Symbol table:\n"; for (auto &KV : Symbols) { @@ -1727,7 +1761,7 @@ JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { JITDylib::JITDylib(ExecutionSession &ES, std::string Name) : ES(ES), JITDylibName(std::move(Name)) { - SearchOrder.push_back({this, true}); + SearchOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols}); } Error JITDylib::defineImpl(MaterializationUnit &MU) { @@ -1820,12 +1854,6 @@ void JITDylib::transferEmittedNodeDependencies( ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) { - // Construct the main dylib. - JDs.push_back(std::unique_ptr<JITDylib>(new JITDylib(*this, "<main>"))); -} - -JITDylib &ExecutionSession::getMainJITDylib() { - return runSessionLocked([this]() -> JITDylib & { return *JDs.front(); }); } JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { @@ -1837,14 +1865,11 @@ JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { }); } -JITDylib &ExecutionSession::createJITDylib(std::string Name, - bool AddToMainDylibSearchOrder) { +JITDylib &ExecutionSession::createJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { JDs.push_back( std::unique_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); - if (AddToMainDylibSearchOrder) - JDs.front()->addToSearchOrder(*JDs.back()); return *JDs.back(); }); } @@ -1895,7 +1920,7 @@ Expected<SymbolMap> ExecutionSession::legacyLookup( #endif auto Query = std::make_shared<AsynchronousSymbolQuery>( - Names, RequiredState, std::move(NotifyComplete)); + SymbolLookupSet(Names), RequiredState, std::move(NotifyComplete)); // FIXME: This should be run session locked along with the registration code // and error reporting below. SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); @@ -1932,8 +1957,9 @@ Expected<SymbolMap> ExecutionSession::legacyLookup( } void ExecutionSession::lookup( - const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, - SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, + LookupKind K, const JITDylibSearchOrder &SearchOrder, + SymbolLookupSet Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies) { LLVM_DEBUG({ @@ -1962,14 +1988,24 @@ void ExecutionSession::lookup( "JITDylibList should not contain duplicate entries"); auto &JD = *KV.first; - auto MatchNonExported = KV.second; - if (auto Err = JD.lodgeQuery(Q, Unresolved, MatchNonExported, - CollectedMUsMap[&JD])) + auto JDLookupFlags = KV.second; + if (auto Err = JD.lodgeQuery(CollectedMUsMap[&JD], Q, K, JDLookupFlags, + Unresolved)) return Err; } + // Strip any weakly referenced symbols that were not found. + Unresolved.forEachWithRemoval( + [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) { + if (Flags == SymbolLookupFlags::WeaklyReferencedSymbol) { + Q->dropSymbol(Name); + return true; + } + return false; + }); + if (!Unresolved.empty()) - return make_error<SymbolsNotFound>(std::move(Unresolved)); + return make_error<SymbolsNotFound>(Unresolved.getSymbolNames()); return Error::success(); }; @@ -2023,8 +2059,8 @@ void ExecutionSession::lookup( } Expected<SymbolMap> -ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, - const SymbolNameSet &Symbols, +ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, + const SymbolLookupSet &Symbols, LookupKind K, SymbolState RequiredState, RegisterDependenciesFunction RegisterDependencies) { #if LLVM_ENABLE_THREADS @@ -2056,7 +2092,7 @@ ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, #endif // Perform the asynchronous lookup. - lookup(SearchOrder, Symbols, RequiredState, NotifyComplete, + lookup(K, SearchOrder, Symbols, RequiredState, NotifyComplete, RegisterDependencies); #if LLVM_ENABLE_THREADS @@ -2077,12 +2113,12 @@ ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, } Expected<JITEvaluatedSymbol> -ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, +ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Name) { - SymbolNameSet Names({Name}); + SymbolLookupSet Names({Name}); - if (auto ResultMap = lookup(SearchOrder, std::move(Names), SymbolState::Ready, - NoDependenciesToRegister)) { + if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static, + SymbolState::Ready, NoDependenciesToRegister)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); @@ -2093,14 +2129,7 @@ ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, Expected<JITEvaluatedSymbol> ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name) { - SymbolNameSet Names({Name}); - - JITDylibSearchList FullSearchOrder; - FullSearchOrder.reserve(SearchOrder.size()); - for (auto *JD : SearchOrder) - FullSearchOrder.push_back({JD, false}); - - return lookup(FullSearchOrder, Name); + return lookup(makeJITDylibSearchOrder(SearchOrder), Name); } Expected<JITEvaluatedSymbol> diff --git a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp new file mode 100644 index 000000000000..c9e87ff737fc --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp @@ -0,0 +1,68 @@ +//===---------- DebugUtils.cpp - Utilities for debugging ORC JITs ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +DumpObjects::DumpObjects(std::string DumpDir, std::string IdentifierOverride) + : DumpDir(std::move(DumpDir)), + IdentifierOverride(std::move(IdentifierOverride)) { + + /// Discard any trailing separators. + while (!this->DumpDir.empty() && + sys::path::is_separator(this->DumpDir.back())) + this->DumpDir.pop_back(); +} + +Expected<std::unique_ptr<MemoryBuffer>> +DumpObjects::operator()(std::unique_ptr<MemoryBuffer> Obj) { + size_t Idx = 1; + + std::string DumpPathStem; + raw_string_ostream(DumpPathStem) + << DumpDir << (DumpDir.empty() ? "" : "/") << getBufferIdentifier(*Obj); + + std::string DumpPath = DumpPathStem + ".o"; + while (sys::fs::exists(DumpPath)) { + DumpPath.clear(); + raw_string_ostream(DumpPath) << DumpPathStem << "." << (++Idx) << ".o"; + } + + LLVM_DEBUG({ + dbgs() << "Dumping object buffer [ " << (const void *)Obj->getBufferStart() + << " -- " << (const void *)(Obj->getBufferEnd() - 1) << " ] to " + << DumpPath << "\n"; + }); + + std::error_code EC; + raw_fd_ostream DumpStream(DumpPath, EC); + if (EC) + return errorCodeToError(EC); + DumpStream.write(Obj->getBufferStart(), Obj->getBufferSize()); + + return std::move(Obj); +} + +StringRef DumpObjects::getBufferIdentifier(MemoryBuffer &B) { + if (!IdentifierOverride.empty()) + return IdentifierOverride; + StringRef Identifier = B.getBufferIdentifier(); + Identifier.consume_back(".o"); + return Identifier; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 4a886ac0597c..3d97fe9eeab1 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -19,6 +19,32 @@ namespace llvm { namespace orc { +int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, + Optional<StringRef> ProgramName) { + std::vector<std::unique_ptr<char[]>> ArgVStorage; + std::vector<char *> ArgV; + + ArgVStorage.reserve(Args.size() + (ProgramName ? 1 : 0)); + ArgV.reserve(Args.size() + 1 + (ProgramName ? 1 : 0)); + + if (ProgramName) { + ArgVStorage.push_back(std::make_unique<char[]>(ProgramName->size() + 1)); + llvm::copy(*ProgramName, &ArgVStorage.back()[0]); + ArgVStorage.back()[ProgramName->size()] = '\0'; + ArgV.push_back(ArgVStorage.back().get()); + } + + for (auto &Arg : Args) { + ArgVStorage.push_back(std::make_unique<char[]>(Arg.size() + 1)); + llvm::copy(Arg, &ArgVStorage.back()[0]); + ArgVStorage.back()[Arg.size()] = '\0'; + ArgV.push_back(ArgVStorage.back().get()); + } + ArgV.push_back(nullptr); + + return Main(Args.size() + !!ProgramName, ArgV.data()); +} + CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) : InitList( GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr), @@ -95,7 +121,7 @@ void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) { JD.getExecutionSession(), (*CtorDtors.begin()).Func->getParent()->getDataLayout()); - for (const auto &CtorDtor : CtorDtors) { + for (auto CtorDtor : CtorDtors) { assert(CtorDtor.Func && CtorDtor.Func->hasName() && "Ctor/Dtor function must be named to be runnable under the JIT"); @@ -118,19 +144,17 @@ void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) { Error CtorDtorRunner::run() { using CtorDtorTy = void (*)(); - SymbolNameSet Names; - - for (auto &KV : CtorDtorsByPriority) { - for (auto &Name : KV.second) { - auto Added = Names.insert(Name).second; - (void)Added; - assert(Added && "Ctor/Dtor names clashed"); - } - } + SymbolLookupSet LookupSet; + for (auto &KV : CtorDtorsByPriority) + for (auto &Name : KV.second) + LookupSet.add(Name); + assert(!LookupSet.containsDuplicates() && + "Ctor/Dtor list contains duplicates"); auto &ES = JD.getExecutionSession(); - if (auto CtorDtorMap = - ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names))) { + if (auto CtorDtorMap = ES.lookup( + makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), + std::move(LookupSet))) { for (auto &KV : CtorDtorsByPriority) { for (auto &Name : KV.second) { assert(CtorDtorMap->count(Name) && "No entry for Name"); @@ -190,15 +214,16 @@ DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, std::move(Lib), GlobalPrefix, std::move(Allow)); } -Expected<SymbolNameSet> -DynamicLibrarySearchGenerator::tryToGenerate(JITDylib &JD, - const SymbolNameSet &Names) { - orc::SymbolNameSet Added; +Error DynamicLibrarySearchGenerator::tryToGenerate( + LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &Symbols) { orc::SymbolMap NewSymbols; bool HasGlobalPrefix = (GlobalPrefix != '\0'); - for (auto &Name : Names) { + for (auto &KV : Symbols) { + auto &Name = KV.first; + if ((*Name).empty()) continue; @@ -211,20 +236,16 @@ DynamicLibrarySearchGenerator::tryToGenerate(JITDylib &JD, std::string Tmp((*Name).data() + HasGlobalPrefix, (*Name).size() - HasGlobalPrefix); if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { - Added.insert(Name); NewSymbols[Name] = JITEvaluatedSymbol( static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)), JITSymbolFlags::Exported); } } - // Add any new symbols to JD. Since the generator is only called for symbols - // that are not already defined, this will never trigger a duplicate - // definition error, so we can wrap this call in a 'cantFail'. - if (!NewSymbols.empty()) - cantFail(JD.define(absoluteSymbols(std::move(NewSymbols)))); + if (NewSymbols.empty()) + return Error::success(); - return Added; + return JD.define(absoluteSymbols(std::move(NewSymbols))); } Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> @@ -251,15 +272,24 @@ StaticLibraryDefinitionGenerator::Create( return std::move(ADG); } -Expected<SymbolNameSet> -StaticLibraryDefinitionGenerator::tryToGenerate(JITDylib &JD, - const SymbolNameSet &Names) { +Error StaticLibraryDefinitionGenerator::tryToGenerate( + LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &Symbols) { + + // Don't materialize symbols from static archives unless this is a static + // lookup. + if (K != LookupKind::Static) + return Error::success(); + + // Bail out early if we've already freed the archive. + if (!Archive) + return Error::success(); DenseSet<std::pair<StringRef, StringRef>> ChildBufferInfos; - SymbolNameSet NewDefs; - for (const auto &Name : Names) { - auto Child = Archive.findSym(*Name); + for (const auto &KV : Symbols) { + const auto &Name = KV.first; + auto Child = Archive->findSym(*Name); if (!Child) return Child.takeError(); if (*Child == None) @@ -269,7 +299,6 @@ StaticLibraryDefinitionGenerator::tryToGenerate(JITDylib &JD, return ChildBuffer.takeError(); ChildBufferInfos.insert( {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()}); - NewDefs.insert(Name); } for (auto ChildBufferInfo : ChildBufferInfos) { @@ -278,31 +307,16 @@ StaticLibraryDefinitionGenerator::tryToGenerate(JITDylib &JD, if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef), VModuleKey())) - return std::move(Err); - - --UnrealizedObjects; + return Err; } - return NewDefs; + return Error::success(); } StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, Error &Err) : L(L), ArchiveBuffer(std::move(ArchiveBuffer)), - Archive(*this->ArchiveBuffer, Err) { - - if (Err) - return; - - Error Err2 = Error::success(); - for (auto _ : Archive.children(Err2)) { - (void)_; - ++UnrealizedObjects; - } - - // No need to check this: We will leave it to the caller. - Err = std::move(Err2); -} + Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) {} } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 0295db7633dd..1ac9a58aeaef 100644 --- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -101,7 +101,10 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( Name = I->second; } - if (auto Sym = ES.lookup(JITDylibSearchList({{&CallbacksJD, true}}), Name)) + if (auto Sym = + ES.lookup(makeJITDylibSearchOrder( + &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), + Name)) return Sym->getAddress(); else { llvm::dbgs() << "Didn't find callback.\n"; @@ -199,7 +202,7 @@ createLocalIndirectStubsManagerBuilder(const Triple &T) { return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcMips64>>(); }; - + case Triple::x86_64: if (T.getOS() == Triple::OSType::Win32) { return [](){ diff --git a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 1d3e6db913e2..114e81e41771 100644 --- a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -28,14 +28,12 @@ Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { // Retrieve host CPU name and sub-target features and add them to builder. // Relocation model, code model and codegen opt level are kept to default // values. - llvm::SubtargetFeatures SubtargetFeatures; llvm::StringMap<bool> FeatureMap; llvm::sys::getHostCPUFeatures(FeatureMap); for (auto &Feature : FeatureMap) - SubtargetFeatures.AddFeature(Feature.first(), Feature.second); + TMBuilder.getFeatures().AddFeature(Feature.first(), Feature.second); TMBuilder.setCPU(llvm::sys::getHostCPUName()); - TMBuilder.addFeatures(SubtargetFeatures.getFeatures()); return TMBuilder; } diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index a80f78afe80f..54473ab46423 100644 --- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -7,6 +7,8 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" @@ -24,6 +26,26 @@ Error LLJITBuilderState::prepareForConstruction() { return JTMBOrErr.takeError(); } + // If the client didn't configure any linker options then auto-configure the + // JIT linker. + if (!CreateObjectLinkingLayer && JTMB->getCodeModel() == None && + JTMB->getRelocationModel() == None) { + + auto &TT = JTMB->getTargetTriple(); + if (TT.isOSBinFormatMachO() && + (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) { + + JTMB->setRelocationModel(Reloc::PIC_); + JTMB->setCodeModel(CodeModel::Small); + CreateObjectLinkingLayer = + [](ExecutionSession &ES, + const Triple &) -> std::unique_ptr<ObjectLayer> { + return std::make_unique<ObjectLinkingLayer>( + ES, std::make_unique<jitlink::InProcessMemoryManager>()); + }; + } + } + return Error::success(); } @@ -51,12 +73,14 @@ Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { assert(Obj && "Can not add null object"); - return ObjLinkingLayer->add(JD, std::move(Obj), ES->allocateVModule()); + return ObjTransformLayer.add(JD, std::move(Obj), ES->allocateVModule()); } Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, StringRef Name) { - return ES->lookup(JITDylibSearchList({{&JD, true}}), ES->intern(Name)); + return ES->lookup( + makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), + ES->intern(Name)); } std::unique_ptr<ObjectLayer> @@ -103,13 +127,13 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), - Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main), + Main(this->ES->createJITDylib("<main>")), DL(""), + ObjLinkingLayer(createObjectLinkingLayer(S, *ES)), + ObjTransformLayer(*this->ES, *ObjLinkingLayer), CtorRunner(Main), DtorRunner(Main) { ErrorAsOutParameter _(&Err); - ObjLinkingLayer = createObjectLinkingLayer(S, *ES); - if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) DL = std::move(*DLOrErr); else { @@ -124,7 +148,7 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) return; } CompileLayer = std::make_unique<IRCompileLayer>( - *ES, *ObjLinkingLayer, std::move(*CompileFunction)); + *ES, ObjTransformLayer, std::move(*CompileFunction)); } if (S.NumCompileThreads > 0) { diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp index 93aabd817d60..aab490feb8ea 100644 --- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -50,8 +50,10 @@ LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { SourceJD = I->second.first; SymbolName = I->second.second; } - auto LookupResult = - ES.lookup(JITDylibSearchList({{SourceJD, true}}), SymbolName); + + auto LookupResult = ES.lookup( + makeJITDylibSearchOrder(SourceJD, JITDylibLookupFlags::MatchAllSymbols), + SymbolName); if (!LookupResult) { ES.reportError(LookupResult.takeError()); diff --git a/llvm/lib/ExecutionEngine/Orc/Legacy.cpp b/llvm/lib/ExecutionEngine/Orc/Legacy.cpp index 9f9a6730b2c3..67b804c37287 100644 --- a/llvm/lib/ExecutionEngine/Orc/Legacy.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Legacy.cpp @@ -37,7 +37,8 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, }; auto Q = std::make_shared<AsynchronousSymbolQuery>( - InternedSymbols, SymbolState::Resolved, std::move(OnResolvedWithUnwrap)); + SymbolLookupSet(InternedSymbols), SymbolState::Resolved, + std::move(OnResolvedWithUnwrap)); auto Unresolved = R.lookup(Q, InternedSymbols); if (Unresolved.empty()) { diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 874decb2ade0..2572b7f4878d 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -36,7 +36,7 @@ public: Layer.ReturnObjectBuffer(std::move(ObjBuffer)); } - JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + JITLinkMemoryManager &getMemoryManager() override { return *Layer.MemMgr; } MemoryBufferRef getObjectBuffer() const override { return ObjBuffer->getMemBufferRef(); @@ -47,18 +47,28 @@ public: MR.failMaterialization(); } - void lookup(const DenseSet<StringRef> &Symbols, + void lookup(const LookupMap &Symbols, std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { - JITDylibSearchList SearchOrder; + JITDylibSearchOrder SearchOrder; MR.getTargetJITDylib().withSearchOrderDo( - [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + [&](const JITDylibSearchOrder &O) { SearchOrder = O; }); auto &ES = Layer.getExecutionSession(); - SymbolNameSet InternedSymbols; - for (auto &S : Symbols) - InternedSymbols.insert(ES.intern(S)); + SymbolLookupSet LookupSet; + for (auto &KV : Symbols) { + orc::SymbolLookupFlags LookupFlags; + switch (KV.second) { + case jitlink::SymbolLookupFlags::RequiredSymbol: + LookupFlags = orc::SymbolLookupFlags::RequiredSymbol; + break; + case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol: + LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol; + break; + } + LookupSet.add(ES.intern(KV.first), LookupFlags); + } // OnResolve -- De-intern the symbols and pass the result to the linker. auto OnResolve = [this, LookupContinuation = std::move(LC)]( @@ -74,8 +84,9 @@ public: } }; - ES.lookup(SearchOrder, std::move(InternedSymbols), SymbolState::Resolved, - std::move(OnResolve), [this](const SymbolDependenceMap &Deps) { + ES.lookup(LookupKind::Static, SearchOrder, std::move(LookupSet), + SymbolState::Resolved, std::move(OnResolve), + [this](const SymbolDependenceMap &Deps) { registerDependencies(Deps); }); } @@ -317,9 +328,9 @@ private: ObjectLinkingLayer::Plugin::~Plugin() {} -ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, - JITLinkMemoryManager &MemMgr) - : ObjectLayer(ES), MemMgr(MemMgr) {} +ObjectLinkingLayer::ObjectLinkingLayer( + ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) + : ObjectLayer(ES), MemMgr(std::move(MemMgr)) {} ObjectLinkingLayer::~ObjectLinkingLayer() { if (auto Err = removeAllModules()) diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp index 815517321b76..d18eb38a4142 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -21,12 +21,18 @@ void ObjectTransformLayer::emit(MaterializationResponsibility R, std::unique_ptr<MemoryBuffer> O) { assert(O && "Module must not be null"); - if (auto TransformedObj = Transform(std::move(O))) - BaseLayer.emit(std::move(R), std::move(*TransformedObj)); - else { - R.failMaterialization(); - getExecutionSession().reportError(TransformedObj.takeError()); + // If there is a transform set then apply it. + if (Transform) { + if (auto TransformedObj = Transform(std::move(O))) + O = std::move(*TransformedObj); + else { + R.failMaterialization(); + getExecutionSession().reportError(TransformedObj.takeError()); + return; + } } + + BaseLayer.emit(std::move(R), std::move(O)); } } // End namespace orc. diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 939cd539d1fb..a92264c0be14 100644 --- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -19,11 +19,11 @@ public: void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) { auto &ES = MR.getTargetJITDylib().getExecutionSession(); - SymbolNameSet InternedSymbols; + SymbolLookupSet InternedSymbols; // Intern the requested symbols: lookup takes interned strings. for (auto &S : Symbols) - InternedSymbols.insert(ES.intern(S)); + InternedSymbols.add(ES.intern(S)); // Build an OnResolve callback to unwrap the interned strings and pass them // to the OnResolved callback. @@ -46,11 +46,12 @@ public: MR.addDependenciesForAll(Deps); }; - JITDylibSearchList SearchOrder; + JITDylibSearchOrder SearchOrder; MR.getTargetJITDylib().withSearchOrderDo( - [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); - ES.lookup(SearchOrder, InternedSymbols, SymbolState::Resolved, - std::move(OnResolvedWithUnwrap), RegisterDependencies); + [&](const JITDylibSearchOrder &JDs) { SearchOrder = JDs; }); + ES.lookup(LookupKind::Static, SearchOrder, InternedSymbols, + SymbolState::Resolved, std::move(OnResolvedWithUnwrap), + RegisterDependencies); } Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) { @@ -77,6 +78,12 @@ RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) {} +RTDyldObjectLinkingLayer::~RTDyldObjectLinkingLayer() { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + for (auto &MemMgr : MemMgrs) + MemMgr->deregisterEHFrames(); +} + void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, std::unique_ptr<MemoryBuffer> O) { assert(O && "Object must not be null"); diff --git a/llvm/lib/ExecutionEngine/Orc/OrcError.cpp b/llvm/lib/ExecutionEngine/OrcError/OrcError.cpp index e6e9a095319c..5eab246d4b48 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcError.cpp +++ b/llvm/lib/ExecutionEngine/OrcError/OrcError.cpp @@ -14,6 +14,8 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" +#include <type_traits> + using namespace llvm; using namespace llvm::orc; diff --git a/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp b/llvm/lib/ExecutionEngine/OrcError/RPCError.cpp index 367b3639f841..3cf78fd9f7ba 100644 --- a/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp +++ b/llvm/lib/ExecutionEngine/OrcError/RPCError.cpp @@ -1,4 +1,4 @@ -//===--------------- RPCUtils.cpp - RPCUtils implementation ---------------===// +//===--------------- RPCError.cpp - RPCERror implementation ---------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,11 +6,16 @@ // //===----------------------------------------------------------------------===// // -// RPCUtils implementation. +// RPC Error type implmentations. // //===----------------------------------------------------------------------===// -#include "llvm/ExecutionEngine/Orc/RPCUtils.h" +#include "llvm/ExecutionEngine/Orc/RPC/RPCUtils.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" + +#include <system_error> +#include <string> char llvm::orc::rpc::RPCFatalError::ID = 0; char llvm::orc::rpc::ConnectionClosed::ID = 0; diff --git a/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index 184388dc4d7a..cc196df3b2fa 100644 --- a/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -24,6 +24,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Errno.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/Path.h" diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index 27a7690db34f..6e3cd7cd2cfc 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "RuntimeDyldCOFF.h" +#include "Targets/RuntimeDyldCOFFAArch64.h" #include "Targets/RuntimeDyldCOFFI386.h" #include "Targets/RuntimeDyldCOFFThumb.h" #include "Targets/RuntimeDyldCOFFX86_64.h" @@ -55,6 +56,8 @@ llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch, return std::make_unique<RuntimeDyldCOFFThumb>(MemMgr, Resolver); case Triple::x86_64: return std::make_unique<RuntimeDyldCOFFX86_64>(MemMgr, Resolver); + case Triple::aarch64: + return std::make_unique<RuntimeDyldCOFFAArch64>(MemMgr, Resolver); } } diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index b9c5a12e08d8..2ac0586ff324 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -667,7 +667,7 @@ private: ArrayRef<uint8_t> SymbolBytes(SymbolMem.bytes_begin(), SymbolMem.size()); MCDisassembler::DecodeStatus S = - Dis->getInstruction(Inst, Size, SymbolBytes, 0, nulls(), nulls()); + Dis->getInstruction(Inst, Size, SymbolBytes, 0, nulls()); return (S == MCDisassembler::Success); } diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h new file mode 100644 index 000000000000..a94f54f50ac4 --- /dev/null +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h @@ -0,0 +1,365 @@ +//===-- RuntimeDyldCOFFAArch64.h --- COFF/AArch64 specific code ---*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// COFF AArch64 support for MC-JIT runtime dynamic linker. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H + +#include "../RuntimeDyldCOFF.h" +#include "llvm/BinaryFormat/COFF.h" +#include "llvm/Object/COFF.h" +#include "llvm/Support/Endian.h" + +#define DEBUG_TYPE "dyld" + +using namespace llvm::support::endian; + +namespace llvm { + +// This relocation type is used for handling long branch instruction +// throught the Stub. +enum InternalRelocationType : unsigned { + INTERNAL_REL_ARM64_LONG_BRANCH26 = 0x111, +}; + +static void add16(uint8_t *p, int16_t v) { write16le(p, read16le(p) + v); } +static void or32le(void *P, int32_t V) { write32le(P, read32le(P) | V); } + +static void write32AArch64Imm(uint8_t *T, uint64_t imm, uint32_t rangeLimit) { + uint32_t orig = read32le(T); + orig &= ~(0xFFF << 10); + write32le(T, orig | ((imm & (0xFFF >> rangeLimit)) << 10)); +} + +static void write32AArch64Ldr(uint8_t *T, uint64_t imm) { + uint32_t orig = read32le(T); + uint32_t size = orig >> 30; + // 0x04000000 indicates SIMD/FP registers + // 0x00800000 indicates 128 bit + if ((orig & 0x04800000) == 0x04800000) + size += 4; + if ((imm & ((1 << size) - 1)) != 0) + assert(0 && "misaligned ldr/str offset"); + write32AArch64Imm(T, imm >> size, size); +} + +static void write32AArch64Addr(void *T, uint64_t s, uint64_t p, int shift) { + uint64_t Imm = (s >> shift) - (p >> shift); + uint32_t ImmLo = (Imm & 0x3) << 29; + uint32_t ImmHi = (Imm & 0x1FFFFC) << 3; + uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3); + write32le(T, (read32le(T) & ~Mask) | ImmLo | ImmHi); +} + +class RuntimeDyldCOFFAArch64 : public RuntimeDyldCOFF { + +private: + // When a module is loaded we save the SectionID of the unwind + // sections in a table until we receive a request to register all + // unregisteredEH frame sections with the memory manager. + SmallVector<SID, 2> UnregisteredEHFrameSections; + SmallVector<SID, 2> RegisteredEHFrameSections; + uint64_t ImageBase; + + // Fake an __ImageBase pointer by returning the section with the lowest adress + uint64_t getImageBase() { + if (!ImageBase) { + ImageBase = std::numeric_limits<uint64_t>::max(); + for (const SectionEntry &Section : Sections) + // The Sections list may contain sections that weren't loaded for + // whatever reason: they may be debug sections, and ProcessAllSections + // is false, or they may be sections that contain 0 bytes. If the + // section isn't loaded, the load address will be 0, and it should not + // be included in the ImageBase calculation. + if (Section.getLoadAddress() != 0) + ImageBase = std::min(ImageBase, Section.getLoadAddress()); + } + return ImageBase; + } + +public: + RuntimeDyldCOFFAArch64(RuntimeDyld::MemoryManager &MM, + JITSymbolResolver &Resolver) + : RuntimeDyldCOFF(MM, Resolver), ImageBase(0) {} + + unsigned getStubAlignment() override { return 8; } + + unsigned getMaxStubSize() const override { return 20; } + + std::tuple<uint64_t, uint64_t, uint64_t> + generateRelocationStub(unsigned SectionID, StringRef TargetName, + uint64_t Offset, uint64_t RelType, uint64_t Addend, + StubMap &Stubs) { + uintptr_t StubOffset; + SectionEntry &Section = Sections[SectionID]; + + RelocationValueRef OriginalRelValueRef; + OriginalRelValueRef.SectionID = SectionID; + OriginalRelValueRef.Offset = Offset; + OriginalRelValueRef.Addend = Addend; + OriginalRelValueRef.SymbolName = TargetName.data(); + + auto Stub = Stubs.find(OriginalRelValueRef); + if (Stub == Stubs.end()) { + LLVM_DEBUG(dbgs() << " Create a new stub function for " + << TargetName.data() << "\n"); + + StubOffset = Section.getStubOffset(); + Stubs[OriginalRelValueRef] = StubOffset; + createStubFunction(Section.getAddressWithOffset(StubOffset)); + Section.advanceStubOffset(getMaxStubSize()); + } else { + LLVM_DEBUG(dbgs() << " Stub function found for " << TargetName.data() + << "\n"); + StubOffset = Stub->second; + } + + // Resolve original relocation to stub function. + const RelocationEntry RE(SectionID, Offset, RelType, Addend); + resolveRelocation(RE, Section.getLoadAddressWithOffset(StubOffset)); + + // adjust relocation info so resolution writes to the stub function + // Here an internal relocation type is used for resolving long branch via + // stub instruction. + Addend = 0; + Offset = StubOffset; + RelType = INTERNAL_REL_ARM64_LONG_BRANCH26; + + return std::make_tuple(Offset, RelType, Addend); + } + + Expected<object::relocation_iterator> + processRelocationRef(unsigned SectionID, object::relocation_iterator RelI, + const object::ObjectFile &Obj, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { + + auto Symbol = RelI->getSymbol(); + if (Symbol == Obj.symbol_end()) + report_fatal_error("Unknown symbol in relocation"); + + Expected<StringRef> TargetNameOrErr = Symbol->getName(); + if (!TargetNameOrErr) + return TargetNameOrErr.takeError(); + StringRef TargetName = *TargetNameOrErr; + + auto SectionOrErr = Symbol->getSection(); + if (!SectionOrErr) + return SectionOrErr.takeError(); + auto Section = *SectionOrErr; + + uint64_t RelType = RelI->getType(); + uint64_t Offset = RelI->getOffset(); + + // If there is no section, this must be an external reference. + const bool IsExtern = Section == Obj.section_end(); + + // Determine the Addend used to adjust the relocation value. + uint64_t Addend = 0; + SectionEntry &AddendSection = Sections[SectionID]; + uintptr_t ObjTarget = AddendSection.getObjAddress() + Offset; + uint8_t *Displacement = (uint8_t *)ObjTarget; + + switch (RelType) { + case COFF::IMAGE_REL_ARM64_ADDR32: + case COFF::IMAGE_REL_ARM64_ADDR32NB: + case COFF::IMAGE_REL_ARM64_REL32: + case COFF::IMAGE_REL_ARM64_SECREL: + Addend = read32le(Displacement); + break; + case COFF::IMAGE_REL_ARM64_BRANCH26: { + uint32_t orig = read32le(Displacement); + Addend = (orig & 0x03FFFFFF) << 2; + + if (IsExtern) + std::tie(Offset, RelType, Addend) = generateRelocationStub( + SectionID, TargetName, Offset, RelType, Addend, Stubs); + break; + } + case COFF::IMAGE_REL_ARM64_BRANCH19: { + uint32_t orig = read32le(Displacement); + Addend = (orig & 0x00FFFFE0) >> 3; + break; + } + case COFF::IMAGE_REL_ARM64_BRANCH14: { + uint32_t orig = read32le(Displacement); + Addend = (orig & 0x000FFFE0) >> 3; + break; + } + case COFF::IMAGE_REL_ARM64_REL21: + case COFF::IMAGE_REL_ARM64_PAGEBASE_REL21: { + uint32_t orig = read32le(Displacement); + Addend = ((orig >> 29) & 0x3) | ((orig >> 3) & 0x1FFFFC); + break; + } + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L: + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A: { + uint32_t orig = read32le(Displacement); + Addend = ((orig >> 10) & 0xFFF); + break; + } + case COFF::IMAGE_REL_ARM64_ADDR64: { + Addend = read64le(Displacement); + break; + } + default: + break; + } + +#if !defined(NDEBUG) + SmallString<32> RelTypeName; + RelI->getTypeName(RelTypeName); + + LLVM_DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset + << " RelType: " << RelTypeName << " TargetName: " + << TargetName << " Addend " << Addend << "\n"); +#endif + + unsigned TargetSectionID = -1; + if (IsExtern) { + RelocationEntry RE(SectionID, Offset, RelType, Addend); + addRelocationForSymbol(RE, TargetName); + } else { + if (auto TargetSectionIDOrErr = findOrEmitSection( + Obj, *Section, Section->isText(), ObjSectionToID)) { + TargetSectionID = *TargetSectionIDOrErr; + } else + return TargetSectionIDOrErr.takeError(); + + uint64_t TargetOffset = getSymbolOffset(*Symbol); + RelocationEntry RE(SectionID, Offset, RelType, TargetOffset + Addend); + addRelocationForSection(RE, TargetSectionID); + } + return ++RelI; + } + + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { + const auto Section = Sections[RE.SectionID]; + uint8_t *Target = Section.getAddressWithOffset(RE.Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); + + switch (RE.RelType) { + default: + llvm_unreachable("unsupported relocation type"); + case COFF::IMAGE_REL_ARM64_ABSOLUTE: { + // This relocation is ignored. + break; + } + case COFF::IMAGE_REL_ARM64_PAGEBASE_REL21: { + // The page base of the target, for ADRP instruction. + Value += RE.Addend; + write32AArch64Addr(Target, Value, FinalAddress, 12); + break; + } + case COFF::IMAGE_REL_ARM64_REL21: { + // The 12-bit relative displacement to the target, for instruction ADR + Value += RE.Addend; + write32AArch64Addr(Target, Value, FinalAddress, 0); + break; + } + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A: { + // The 12-bit page offset of the target, + // for instructions ADD/ADDS (immediate) with zero shift. + Value += RE.Addend; + write32AArch64Imm(Target, Value & 0xFFF, 0); + break; + } + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L: { + // The 12-bit page offset of the target, + // for instruction LDR (indexed, unsigned immediate). + Value += RE.Addend; + write32AArch64Ldr(Target, Value & 0xFFF); + break; + } + case COFF::IMAGE_REL_ARM64_ADDR32: { + // The 32-bit VA of the target. + uint32_t VA = Value + RE.Addend; + write32le(Target, VA); + break; + } + case COFF::IMAGE_REL_ARM64_ADDR32NB: { + // The target's 32-bit RVA. + uint64_t RVA = Value + RE.Addend - getImageBase(); + write32le(Target, RVA); + break; + } + case INTERNAL_REL_ARM64_LONG_BRANCH26: { + // Encode the immadiate value for generated Stub instruction (MOVZ) + or32le(Target + 12, ((Value + RE.Addend) & 0xFFFF) << 5); + or32le(Target + 8, ((Value + RE.Addend) & 0xFFFF0000) >> 11); + or32le(Target + 4, ((Value + RE.Addend) & 0xFFFF00000000) >> 27); + or32le(Target + 0, ((Value + RE.Addend) & 0xFFFF000000000000) >> 43); + break; + } + case COFF::IMAGE_REL_ARM64_BRANCH26: { + // The 26-bit relative displacement to the target, for B and BL + // instructions. + uint64_t PCRelVal = Value + RE.Addend - FinalAddress; + assert(isInt<28>(PCRelVal) && "Branch target is out of range."); + write32le(Target, (read32le(Target) & ~(0x03FFFFFF)) | + (PCRelVal & 0x0FFFFFFC) >> 2); + break; + } + case COFF::IMAGE_REL_ARM64_BRANCH19: { + // The 19-bit offset to the relocation target, + // for conditional B instruction. + uint64_t PCRelVal = Value + RE.Addend - FinalAddress; + assert(isInt<21>(PCRelVal) && "Branch target is out of range."); + write32le(Target, (read32le(Target) & ~(0x00FFFFE0)) | + (PCRelVal & 0x001FFFFC) << 3); + break; + } + case COFF::IMAGE_REL_ARM64_BRANCH14: { + // The 14-bit offset to the relocation target, + // for instructions TBZ and TBNZ. + uint64_t PCRelVal = Value + RE.Addend - FinalAddress; + assert(isInt<16>(PCRelVal) && "Branch target is out of range."); + write32le(Target, (read32le(Target) & ~(0x000FFFE0)) | + (PCRelVal & 0x0000FFFC) << 3); + break; + } + case COFF::IMAGE_REL_ARM64_ADDR64: { + // The 64-bit VA of the relocation target. + write64le(Target, Value + RE.Addend); + break; + } + case COFF::IMAGE_REL_ARM64_SECTION: { + // 16-bit section index of the section that contains the target. + assert(static_cast<uint32_t>(RE.SectionID) <= UINT16_MAX && + "relocation overflow"); + add16(Target, RE.SectionID); + break; + } + case COFF::IMAGE_REL_ARM64_SECREL: { + // 32-bit offset of the target from the beginning of its section. + assert(static_cast<int64_t>(RE.Addend) <= INT32_MAX && + "Relocation overflow"); + assert(static_cast<int64_t>(RE.Addend) >= INT32_MIN && + "Relocation underflow"); + write32le(Target, RE.Addend); + break; + } + case COFF::IMAGE_REL_ARM64_REL32: { + // The 32-bit relative address from the byte following the relocation. + uint64_t Result = Value - FinalAddress - 4; + write32le(Target, Result + RE.Addend); + break; + } + } + } + + void registerEHFrames() override {} +}; + +} // End namespace llvm + +#endif diff --git a/llvm/lib/ExecutionEngine/TargetSelect.cpp b/llvm/lib/ExecutionEngine/TargetSelect.cpp index 0d9c6cfa0908..28ea04be1a5e 100644 --- a/llvm/lib/ExecutionEngine/TargetSelect.cpp +++ b/llvm/lib/ExecutionEngine/TargetSelect.cpp @@ -83,13 +83,6 @@ TargetMachine *EngineBuilder::selectTarget(const Triple &TargetTriple, FeaturesStr = Features.getString(); } - // FIXME: non-iOS ARM FastISel is broken with MCJIT. - if (TheTriple.getArch() == Triple::arm && - !TheTriple.isiOS() && - OptLevel == CodeGenOpt::None) { - OptLevel = CodeGenOpt::Less; - } - // Allocate a target... TargetMachine *Target = TheTarget->createTargetMachine(TheTriple.getTriple(), MCPU, FeaturesStr, |