diff options
Diffstat (limited to 'lib/ExecutionEngine')
48 files changed, 3620 insertions, 1774 deletions
diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp index 1c6c0406d048..ee7a7cb60bc9 100644 --- a/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/lib/ExecutionEngine/ExecutionEngine.cpp @@ -32,12 +32,12 @@ #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Host.h" -#include "llvm/Support/MutexGuard.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include <cmath> #include <cstring> +#include <mutex> using namespace llvm; #define DEBUG_TYPE "jit" @@ -191,7 +191,7 @@ uint64_t ExecutionEngineState::RemoveMapping(StringRef Name) { std::string ExecutionEngine::getMangledName(const GlobalValue *GV) { assert(GV->hasName() && "Global must have name."); - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); SmallString<128> FullName; const DataLayout &DL = @@ -204,12 +204,12 @@ std::string ExecutionEngine::getMangledName(const GlobalValue *GV) { } void ExecutionEngine::addGlobalMapping(const GlobalValue *GV, void *Addr) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); addGlobalMapping(getMangledName(GV), (uint64_t) Addr); } void ExecutionEngine::addGlobalMapping(StringRef Name, uint64_t Addr) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); assert(!Name.empty() && "Empty GlobalMapping symbol name!"); @@ -228,14 +228,14 @@ void ExecutionEngine::addGlobalMapping(StringRef Name, uint64_t Addr) { } void ExecutionEngine::clearAllGlobalMappings() { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); EEState.getGlobalAddressMap().clear(); EEState.getGlobalAddressReverseMap().clear(); } void ExecutionEngine::clearGlobalMappingsFromModule(Module *M) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); for (GlobalObject &GO : M->global_objects()) EEState.RemoveMapping(getMangledName(&GO)); @@ -243,12 +243,12 @@ void ExecutionEngine::clearGlobalMappingsFromModule(Module *M) { uint64_t ExecutionEngine::updateGlobalMapping(const GlobalValue *GV, void *Addr) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); return updateGlobalMapping(getMangledName(GV), (uint64_t) Addr); } uint64_t ExecutionEngine::updateGlobalMapping(StringRef Name, uint64_t Addr) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); ExecutionEngineState::GlobalAddressMapTy &Map = EEState.getGlobalAddressMap(); @@ -275,7 +275,7 @@ uint64_t ExecutionEngine::updateGlobalMapping(StringRef Name, uint64_t Addr) { } uint64_t ExecutionEngine::getAddressToGlobalIfAvailable(StringRef S) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); uint64_t Address = 0; ExecutionEngineState::GlobalAddressMapTy::iterator I = EEState.getGlobalAddressMap().find(S); @@ -286,19 +286,19 @@ uint64_t ExecutionEngine::getAddressToGlobalIfAvailable(StringRef S) { void *ExecutionEngine::getPointerToGlobalIfAvailable(StringRef S) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); if (void* Address = (void *) getAddressToGlobalIfAvailable(S)) return Address; return nullptr; } void *ExecutionEngine::getPointerToGlobalIfAvailable(const GlobalValue *GV) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); return getPointerToGlobalIfAvailable(getMangledName(GV)); } const GlobalValue *ExecutionEngine::getGlobalValueAtAddress(void *Addr) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // If we haven't computed the reverse mapping yet, do so first. if (EEState.getGlobalAddressReverseMap().empty()) { @@ -340,14 +340,14 @@ void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE, Values.clear(); // Free the old contents. Values.reserve(InputArgv.size()); unsigned PtrSize = EE->getDataLayout().getPointerSize(); - Array = make_unique<char[]>((InputArgv.size()+1)*PtrSize); + Array = std::make_unique<char[]>((InputArgv.size()+1)*PtrSize); LLVM_DEBUG(dbgs() << "JIT: ARGV = " << (void *)Array.get() << "\n"); Type *SBytePtr = Type::getInt8PtrTy(C); for (unsigned i = 0; i != InputArgv.size(); ++i) { unsigned Size = InputArgv[i].size()+1; - auto Dest = make_unique<char[]>(Size); + auto Dest = std::make_unique<char[]>(Size); LLVM_DEBUG(dbgs() << "JIT: ARGV[" << i << "] = " << (void *)Dest.get() << "\n"); @@ -575,7 +575,7 @@ void *ExecutionEngine::getPointerToGlobal(const GlobalValue *GV) { if (Function *F = const_cast<Function*>(dyn_cast<Function>(GV))) return getPointerToFunction(F); - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); if (void* P = getPointerToGlobalIfAvailable(GV)) return P; @@ -626,7 +626,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { break; case Type::VectorTyID: // if the whole vector is 'undef' just reserve memory for the value. - auto* VTy = dyn_cast<VectorType>(C->getType()); + auto* VTy = cast<VectorType>(C->getType()); Type *ElemTy = VTy->getElementType(); unsigned int elemNum = VTy->getNumElements(); Result.AggregateVal.resize(elemNum); @@ -925,7 +925,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { elemNum = CDV->getNumElements(); ElemTy = CDV->getElementType(); } else if (CV || CAZ) { - VectorType* VTy = dyn_cast<VectorType>(C->getType()); + auto* VTy = cast<VectorType>(C->getType()); elemNum = VTy->getNumElements(); ElemTy = VTy->getElementType(); } else { diff --git a/lib/ExecutionEngine/GDBRegistrationListener.cpp b/lib/ExecutionEngine/GDBRegistrationListener.cpp index 08d20156a590..7ed025fbb481 100644 --- a/lib/ExecutionEngine/GDBRegistrationListener.cpp +++ b/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -14,7 +14,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Mutex.h" -#include "llvm/Support/MutexGuard.h" +#include <mutex> using namespace llvm; using namespace llvm::object; @@ -135,7 +135,7 @@ void NotifyDebugger(jit_code_entry* JITCodeEntry) { GDBJITRegistrationListener::~GDBJITRegistrationListener() { // Free all registered object files. - llvm::MutexGuard locked(*JITDebugLock); + std::lock_guard<llvm::sys::Mutex> locked(*JITDebugLock); for (RegisteredObjectBufferMap::iterator I = ObjectBufferMap.begin(), E = ObjectBufferMap.end(); I != E; ++I) { @@ -159,7 +159,7 @@ void GDBJITRegistrationListener::notifyObjectLoaded( const char *Buffer = DebugObj.getBinary()->getMemoryBufferRef().getBufferStart(); size_t Size = DebugObj.getBinary()->getMemoryBufferRef().getBufferSize(); - llvm::MutexGuard locked(*JITDebugLock); + std::lock_guard<llvm::sys::Mutex> locked(*JITDebugLock); assert(ObjectBufferMap.find(K) == ObjectBufferMap.end() && "Second attempt to perform debug registration."); jit_code_entry* JITCodeEntry = new jit_code_entry(); @@ -178,7 +178,7 @@ void GDBJITRegistrationListener::notifyObjectLoaded( } void GDBJITRegistrationListener::notifyFreeingObject(ObjectKey K) { - llvm::MutexGuard locked(*JITDebugLock); + std::lock_guard<llvm::sys::Mutex> locked(*JITDebugLock); RegisteredObjectBufferMap::iterator I = ObjectBufferMap.find(K); if (I != ObjectBufferMap.end()) { diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index c3a2ccc582c9..71b7f893d712 100644 --- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -32,7 +32,6 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Mutex.h" -#include "llvm/Support/UniqueLock.h" #include "llvm/Support/raw_ostream.h" #include <cassert> #include <cmath> @@ -41,6 +40,7 @@ #include <cstdio> #include <cstring> #include <map> +#include <mutex> #include <string> #include <utility> #include <vector> @@ -258,7 +258,7 @@ GenericValue Interpreter::callExternalFunction(Function *F, ArrayRef<GenericValue> ArgVals) { TheInterpreter = this; - unique_lock<sys::Mutex> Guard(*FunctionsLock); + std::unique_lock<sys::Mutex> Guard(*FunctionsLock); // Do a lookup to see if the function is in our cache... this should just be a // deferred annotation! diff --git a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h index 1271ad962b38..b47a798c7603 100644 --- a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h +++ b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -20,24 +20,23 @@ namespace jitlink { template <typename BuilderImpl> class BasicGOTAndStubsBuilder { public: - BasicGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {} void run() { - // We're going to be adding new atoms, but we don't want to iterate over - // the newly added ones, so just copy the existing atoms out. - std::vector<DefinedAtom *> DAs(G.defined_atoms().begin(), - G.defined_atoms().end()); + // We're going to be adding new blocks, but we don't want to iterate over + // the newly added ones, so just copy the existing blocks out. + std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end()); - for (auto *DA : DAs) - for (auto &E : DA->edges()) + for (auto *B : Blocks) + for (auto &E : B->edges()) if (impl().isGOTEdge(E)) - impl().fixGOTEdge(E, getGOTEntryAtom(E.getTarget())); + impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget())); else if (impl().isExternalBranchEdge(E)) - impl().fixExternalBranchEdge(E, getStubAtom(E.getTarget())); + impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget())); } protected: - Atom &getGOTEntryAtom(Atom &Target) { + Symbol &getGOTEntrySymbol(Symbol &Target) { assert(Target.hasName() && "GOT edge cannot point to anonymous target"); auto GOTEntryI = GOTEntries.find(Target.getName()); @@ -49,31 +48,31 @@ protected: GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; } - assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); return *GOTEntryI->second; } - Atom &getStubAtom(Atom &Target) { + Symbol &getStubSymbol(Symbol &Target) { assert(Target.hasName() && "External branch edge can not point to an anonymous target"); auto StubI = Stubs.find(Target.getName()); if (StubI == Stubs.end()) { - auto &StubAtom = impl().createStub(Target); - StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + auto &StubSymbol = impl().createStub(Target); + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; } - assert(StubI != Stubs.end() && "Count not get stub atom"); + assert(StubI != Stubs.end() && "Count not get stub symbol"); return *StubI->second; } - AtomGraph &G; + LinkGraph &G; private: BuilderImpl &impl() { return static_cast<BuilderImpl &>(*this); } - DenseMap<StringRef, DefinedAtom *> GOTEntries; - DenseMap<StringRef, DefinedAtom *> Stubs; + DenseMap<StringRef, Symbol *> GOTEntries; + DenseMap<StringRef, Symbol *> Stubs; }; } // end namespace jitlink diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 25f0e9040ffe..f80b0e7f8909 100644 --- a/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -17,18 +17,14 @@ namespace llvm { namespace jitlink { -EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, - JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) - : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), - EHFrameAddress(EHFrameAddress), - EHFrameReader(EHFrameContent, G.getEndianness()), - FDEToCIERelocKind(FDEToCIERelocKind), - FDEToTargetRelocKind(FDEToTargetRelocKind) {} - -Error EHFrameParser::atomize() { +EHFrameBinaryParser::EHFrameBinaryParser(JITTargetAddress EHFrameAddress, + StringRef EHFrameContent, + unsigned PointerSize, + support::endianness Endianness) + : EHFrameAddress(EHFrameAddress), EHFrameContent(EHFrameContent), + PointerSize(PointerSize), EHFrameReader(EHFrameContent, Endianness) {} + +Error EHFrameBinaryParser::addToGraph() { while (!EHFrameReader.empty()) { size_t RecordOffset = EHFrameReader.getOffset(); @@ -38,44 +34,39 @@ Error EHFrameParser::atomize() { << " (offset " << RecordOffset << ")\n"; }); - size_t CIELength = 0; - uint32_t CIELengthField; - if (auto Err = EHFrameReader.readInteger(CIELengthField)) + size_t RecordLength = 0; + uint32_t RecordLengthField; + if (auto Err = EHFrameReader.readInteger(RecordLengthField)) return Err; - // Process CIE length/extended-length fields to build the atom. + // Process CIE/FDE length/extended-length fields to build the blocks. // // The value of these fields describe the length of the *rest* of the CIE // (not including data up to the end of the field itself) so we have to - // bump CIELength to include the data up to the end of the field: 4 bytes + // bump RecordLength to include the data up to the end of the field: 4 bytes // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. - if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + if (RecordLengthField == 0) // Length 0 means end of __eh_frame section. break; // If the regular length field's value is 0xffffffff, use extended length. - if (CIELengthField == 0xffffffff) { - uint64_t CIEExtendedLengthField; - if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + if (RecordLengthField == 0xffffffff) { + uint64_t ExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(ExtendedLengthField)) return Err; - if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + if (ExtendedLengthField > EHFrameReader.bytesRemaining()) return make_error<JITLinkError>("CIE record extends past the end of " "the __eh_frame section"); - if (CIEExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) + if (ExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) return make_error<JITLinkError>("CIE record too large to process"); - CIELength = CIEExtendedLengthField + 12; + RecordLength = ExtendedLengthField + 12; } else { - if (CIELengthField > EHFrameReader.bytesRemaining()) + if (RecordLengthField > EHFrameReader.bytesRemaining()) return make_error<JITLinkError>("CIE record extends past the end of " "the __eh_frame section"); - CIELength = CIELengthField + 4; + RecordLength = RecordLengthField + 4; } - LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); - - // Add an atom for this record. - CurRecordAtom = &G.addAnonymousAtom( - EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); - CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength)); + LLVM_DEBUG(dbgs() << " length: " << RecordLength << "\n"); // Read the CIE Pointer. size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); @@ -85,21 +76,24 @@ Error EHFrameParser::atomize() { // Based on the CIE pointer value, parse this as a CIE or FDE record. if (CIEPointer == 0) { - if (auto Err = processCIE()) + if (auto Err = processCIE(RecordOffset, RecordLength)) return Err; } else { - if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + if (auto Err = processFDE(RecordOffset, RecordLength, CIEPointerAddress, + CIEPointer)) return Err; } - EHFrameReader.setOffset(RecordOffset + CIELength); + EHFrameReader.setOffset(RecordOffset + RecordLength); } return Error::success(); } -Expected<EHFrameParser::AugmentationInfo> -EHFrameParser::parseAugmentationString() { +void EHFrameBinaryParser::anchor() {} + +Expected<EHFrameBinaryParser::AugmentationInfo> +EHFrameBinaryParser::parseAugmentationString() { AugmentationInfo AugInfo; uint8_t NextChar; uint8_t *NextField = &AugInfo.Fields[0]; @@ -139,14 +133,14 @@ EHFrameParser::parseAugmentationString() { return std::move(AugInfo); } -Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() { +Expected<JITTargetAddress> EHFrameBinaryParser::readAbsolutePointer() { static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), "Result must be able to hold a uint64_t"); JITTargetAddress Addr; - if (G.getPointerSize() == 8) { + if (PointerSize == 8) { if (auto Err = EHFrameReader.readInteger(Addr)) return std::move(Err); - } else if (G.getPointerSize() == 4) { + } else if (PointerSize == 4) { uint32_t Addr32; if (auto Err = EHFrameReader.readInteger(Addr32)) return std::move(Err); @@ -156,14 +150,19 @@ Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() { return Addr; } -Error EHFrameParser::processCIE() { +Error EHFrameBinaryParser::processCIE(size_t RecordOffset, + size_t RecordLength) { // Use the dwarf namespace for convenient access to pointer encoding // constants. using namespace dwarf; LLVM_DEBUG(dbgs() << " Record is CIE\n"); - CIEInformation CIEInfo(*CurRecordAtom); + auto &CIESymbol = + createCIERecord(EHFrameAddress + RecordOffset, + EHFrameContent.substr(RecordOffset, RecordLength)); + + CIEInformation CIEInfo(CIESymbol); uint8_t Version = 0; if (auto Err = EHFrameReader.readInteger(Version)) @@ -179,7 +178,7 @@ Error EHFrameParser::processCIE() { // Skip the EH Data field if present. if (AugInfo->EHDataFieldPresent) - if (auto Err = EHFrameReader.skip(G.getPointerSize())) + if (auto Err = EHFrameReader.skip(PointerSize)) return Err; // Read and sanity check the code alignment factor. @@ -226,7 +225,7 @@ Error EHFrameParser::processCIE() { return make_error<JITLinkError>( "Unsupported LSDA pointer encoding " + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); break; } case 'P': { @@ -239,7 +238,7 @@ Error EHFrameParser::processCIE() { "Unspported personality pointer " "encoding " + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); uint32_t PersonalityPointerAddress; if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) return Err; @@ -254,7 +253,7 @@ Error EHFrameParser::processCIE() { "Unsupported FDE address pointer " "encoding " + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); break; } default: @@ -267,15 +266,16 @@ Error EHFrameParser::processCIE() { return make_error<JITLinkError>("Read past the end of the augmentation " "data while parsing fields"); - assert(!CIEInfos.count(CurRecordAtom->getAddress()) && + assert(!CIEInfos.count(CIESymbol.getAddress()) && "Multiple CIEs recorded at the same address?"); - CIEInfos[CurRecordAtom->getAddress()] = std::move(CIEInfo); + CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); return Error::success(); } -Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, - uint32_t CIEPointer) { +Error EHFrameBinaryParser::processFDE(size_t RecordOffset, size_t RecordLength, + JITTargetAddress CIEPointerAddress, + uint32_t CIEPointer) { LLVM_DEBUG(dbgs() << " Record is FDE\n"); LLVM_DEBUG({ @@ -286,16 +286,11 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer); if (CIEInfoItr == CIEInfos.end()) return make_error<JITLinkError>( - "FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()) + + "FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset) + " points to non-existant CIE at " + formatv("{0:x16}", CIEPointerAddress - CIEPointer)); auto &CIEInfo = CIEInfoItr->second; - // The CIEPointer looks good. Add a relocation. - CurRecordAtom->addEdge(FDEToCIERelocKind, - CIEPointerAddress - CurRecordAtom->getAddress(), - *CIEInfo.CIEAtom, 0); - // Read and sanity check the PC-start pointer and size. JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); @@ -305,83 +300,68 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; LLVM_DEBUG({ - dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; + dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; }); - auto *TargetAtom = G.getAtomByAddress(PCBegin); + auto *TargetSymbol = getSymbolAtAddress(PCBegin); - if (!TargetAtom) + if (!TargetSymbol) return make_error<JITLinkError>("FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point at atom"); + " does not point at symbol"); - if (TargetAtom->getAddress() != PCBegin) + if (TargetSymbol->getAddress() != PCBegin) return make_error<JITLinkError>( "FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point to start of atom at " + - formatv("{0:x16}", TargetAtom->getAddress())); - - LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); + " does not point to start of symbol at " + + formatv("{0:x16}", TargetSymbol->getAddress())); - // The PC-start pointer and size look good. Add relocations. - CurRecordAtom->addEdge(FDEToTargetRelocKind, - PCBeginAddress - CurRecordAtom->getAddress(), - *TargetAtom, 0); - - // Add a keep-alive relocation from the function to the FDE to ensure it is - // not dead stripped. - TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetSymbol << "\n"); // Skip over the PC range size field. - if (auto Err = EHFrameReader.skip(G.getPointerSize())) + if (auto Err = EHFrameReader.skip(PointerSize)) return Err; + Symbol *LSDASymbol = nullptr; + JITTargetAddress LSDAAddress = 0; if (CIEInfo.FDEsHaveLSDAField) { uint64_t AugmentationDataSize; if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) return Err; - if (AugmentationDataSize != G.getPointerSize()) + if (AugmentationDataSize != PointerSize) return make_error<JITLinkError>( "Unexpected FDE augmentation data size (expected " + - Twine(G.getPointerSize()) + ", got " + Twine(AugmentationDataSize) + - ") for FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress())); - JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); + Twine(PointerSize) + ", got " + Twine(AugmentationDataSize) + + ") for FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset)); + LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); auto LSDADelta = readAbsolutePointer(); if (!LSDADelta) return LSDADelta.takeError(); JITTargetAddress LSDA = LSDAAddress + *LSDADelta; - auto *LSDAAtom = G.getAtomByAddress(LSDA); + LSDASymbol = getSymbolAtAddress(LSDA); - if (!LSDAAtom) + if (!LSDASymbol) return make_error<JITLinkError>("FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point at atom"); + " does not point at symbol"); - if (LSDAAtom->getAddress() != LSDA) + if (LSDASymbol->getAddress() != LSDA) return make_error<JITLinkError>( "FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point to start of atom at " + - formatv("{0:x16}", LSDAAtom->getAddress())); - - LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n"); + " does not point to start of symbol at " + + formatv("{0:x16}", LSDASymbol->getAddress())); - // LSDA looks good. Add relocations. - CurRecordAtom->addEdge(FDEToTargetRelocKind, - LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom, - 0); + LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDASymbol << "\n"); } - return Error::success(); -} + JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset; + auto FDESymbol = createFDERecord( + RecordAddress, EHFrameContent.substr(RecordOffset, RecordLength), + *CIEInfo.CIESymbol, CIEPointerAddress - RecordAddress, *TargetSymbol, + PCBeginAddress - RecordAddress, LSDASymbol, LSDAAddress - RecordAddress); -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) { - return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, - FDEToCIERelocKind, FDEToTargetRelocKind) - .atomize(); + return FDESymbol.takeError(); } // Determine whether we can register EH tables. @@ -451,11 +431,13 @@ static Error deregisterFrameWrapper(const void *P) { template <typename HandleFDEFn> Error walkAppleEHFrameSection(const char *const SectionStart, + size_t SectionSize, HandleFDEFn HandleFDE) { const char *CurCFIRecord = SectionStart; + const char *End = SectionStart + SectionSize; uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); - while (Size != 0) { + while (CurCFIRecord != End && Size != 0) { const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); if (Size == 0xffffffff) Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; @@ -484,10 +466,12 @@ Error walkAppleEHFrameSection(const char *const SectionStart, #endif // __APPLE__ -Error registerEHFrameSection(const void *EHFrameSectionAddr) { +Error registerEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize) { #ifdef __APPLE__ // On Darwin __register_frame has to be called for each FDE entry. return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + EHFrameSectionSize, registerFrameWrapper); #else // On Linux __register_frame takes a single argument: @@ -499,9 +483,11 @@ Error registerEHFrameSection(const void *EHFrameSectionAddr) { #endif } -Error deregisterEHFrameSection(const void *EHFrameSectionAddr) { +Error deregisterEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize) { #ifdef __APPLE__ return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + EHFrameSectionSize, deregisterFrameWrapper); #else return deregisterFrameWrapper(EHFrameSectionAddr); @@ -517,23 +503,31 @@ InProcessEHFrameRegistrar &InProcessEHFrameRegistrar::getInstance() { InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {} -AtomGraphPassFunction +LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, - StoreFrameAddressFunction StoreFrameAddress) { + StoreFrameRangeFunction StoreRangeAddress) { const char *EHFrameSectionName = nullptr; if (TT.getObjectFormat() == Triple::MachO) EHFrameSectionName = "__eh_frame"; else EHFrameSectionName = ".eh_frame"; - auto RecordEHFrame = [EHFrameSectionName, - StoreFrameAddress](AtomGraph &G) -> Error { - // Search for a non-empty eh-frame and record the address of the first atom - // in it. + auto RecordEHFrame = + [EHFrameSectionName, + StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first + // symbol in it. JITTargetAddress Addr = 0; - if (auto *S = G.findSectionByName(EHFrameSectionName)) - Addr = S->getRange().getStart(); - StoreFrameAddress(Addr); + size_t Size = 0; + if (auto *S = G.findSectionByName(EHFrameSectionName)) { + auto R = SectionRange(*S); + Addr = R.getStart(); + Size = R.getSize(); + } + if (Addr == 0 && Size != 0) + return make_error<JITLinkError>("__eh_frame section can not have zero " + "address with non-zero size"); + StoreFrameRange(Addr, Size); return Error::success(); }; diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index d679edef7ea6..6f9f68ad8382 100644 --- a/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,18 +21,31 @@ namespace llvm { namespace jitlink { -/// A generic parser for eh-frame sections. +/// A generic binary parser for eh-frame sections. /// -/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and -/// FDEToTarget relocation kinds. -class EHFrameParser { +/// Adds blocks and symbols representing CIE and FDE entries to a JITLink graph. +/// +/// This parser assumes that the user has already verified that the EH-frame's +/// address range does not overlap any other section/symbol, so that generated +/// CIE/FDE records do not overlap other sections/symbols. +class EHFrameBinaryParser { public: - EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, - JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind); - Error atomize(); + EHFrameBinaryParser(JITTargetAddress EHFrameAddress, StringRef EHFrameContent, + unsigned PointerSize, support::endianness Endianness); + virtual ~EHFrameBinaryParser() {} + + Error addToGraph(); private: + virtual void anchor(); + virtual Symbol *getSymbolAtAddress(JITTargetAddress Addr) = 0; + virtual Symbol &createCIERecord(JITTargetAddress RecordAddr, + StringRef RecordContent) = 0; + virtual Expected<Symbol &> + createFDERecord(JITTargetAddress RecordAddr, StringRef RecordContent, + Symbol &CIE, size_t CIEOffset, Symbol &Func, + size_t FuncOffset, Symbol *LSDA, size_t LSDAOffset) = 0; + struct AugmentationInfo { bool AugmentationDataPresent = false; bool EHDataFieldPresent = false; @@ -41,31 +54,24 @@ private: Expected<AugmentationInfo> parseAugmentationString(); Expected<JITTargetAddress> readAbsolutePointer(); - Error processCIE(); - Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer); + Error processCIE(size_t RecordOffset, size_t RecordLength); + Error processFDE(size_t RecordOffset, size_t RecordLength, + JITTargetAddress CIEPointerOffset, uint32_t CIEPointer); struct CIEInformation { CIEInformation() = default; - CIEInformation(DefinedAtom &CIEAtom) : CIEAtom(&CIEAtom) {} - DefinedAtom *CIEAtom = nullptr; + CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} + Symbol *CIESymbol = nullptr; bool FDEsHaveLSDAField = false; }; - AtomGraph &G; - Section &EHFrameSection; - StringRef EHFrameContent; JITTargetAddress EHFrameAddress; + StringRef EHFrameContent; + unsigned PointerSize; BinaryStreamReader EHFrameReader; - DefinedAtom *CurRecordAtom = nullptr; DenseMap<JITTargetAddress, CIEInformation> CIEInfos; - Edge::Kind FDEToCIERelocKind; - Edge::Kind FDEToTargetRelocKind; }; -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); - } // end namespace jitlink } // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/JITLink.cpp b/lib/ExecutionEngine/JITLink/JITLink.cpp index 9d0a7459dc09..1e19038951ac 100644 --- a/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -56,95 +56,151 @@ std::error_code JITLinkError::convertToErrorCode() const { return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); } -const StringRef getGenericEdgeKindName(Edge::Kind K) { +const char *getGenericEdgeKindName(Edge::Kind K) { switch (K) { case Edge::Invalid: return "INVALID RELOCATION"; case Edge::KeepAlive: return "Keep-Alive"; - case Edge::LayoutNext: - return "Layout-Next"; default: llvm_unreachable("Unrecognized relocation kind"); } } -raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { +const char *getLinkageName(Linkage L) { + switch (L) { + case Linkage::Strong: + return "strong"; + case Linkage::Weak: + return "weak"; + } + llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum"); +} + +const char *getScopeName(Scope S) { + switch (S) { + case Scope::Default: + return "default"; + case Scope::Hidden: + return "hidden"; + case Scope::Local: + return "local"; + } + llvm_unreachable("Unrecognized llvm.jitlink.Scope enum"); +} + +raw_ostream &operator<<(raw_ostream &OS, const Block &B) { + return OS << formatv("{0:x16}", B.getAddress()) << " -- " + << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": " + << (B.isZeroFill() ? "zero-fill" : "content") + << ", align = " << B.getAlignment() + << ", align-ofs = " << B.getAlignmentOffset() + << ", section = " << B.getSection().getName(); +} + +raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { OS << "<"; - if (A.getName().empty()) - OS << "anon@" << format("0x%016" PRIx64, A.getAddress()); + if (Sym.getName().empty()) + OS << "*anon*"; else - OS << A.getName(); - OS << " ["; - if (A.isDefined()) { - auto &DA = static_cast<const DefinedAtom &>(A); - OS << " section=" << DA.getSection().getName(); - if (DA.isLive()) - OS << " live"; - if (DA.shouldDiscard()) - OS << " should-discard"; - } else - OS << " external"; - OS << " ]>"; + OS << Sym.getName(); + OS << ": flags = "; + switch (Sym.getLinkage()) { + case Linkage::Strong: + OS << 'S'; + break; + case Linkage::Weak: + OS << 'W'; + break; + } + switch (Sym.getScope()) { + case Scope::Default: + OS << 'D'; + break; + case Scope::Hidden: + OS << 'H'; + break; + case Scope::Local: + OS << 'L'; + break; + } + OS << (Sym.isLive() ? '+' : '-') + << ", size = " << formatv("{0:x8}", Sym.getSize()) + << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " (" + << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + " + << formatv("{0:x8}", Sym.getOffset()); + if (Sym.isDefined()) + OS << " " << Sym.getBlock().getSection().getName(); + OS << ")>"; return OS; } -void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, StringRef EdgeKindName) { - OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) - << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName - << " -> " << E.getTarget() << " + " << E.getAddend(); + OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": " + << formatv("{0:x16}", B.getAddress()) << " + " << E.getOffset() << " -- " + << EdgeKindName << " -> " << E.getTarget() << " + " << E.getAddend(); } Section::~Section() { - for (auto *DA : DefinedAtoms) - DA->~DefinedAtom(); + for (auto *Sym : Symbols) + Sym->~Symbol(); } -void AtomGraph::dump(raw_ostream &OS, +LinkGraph::~LinkGraph() { + // Destroy blocks. + for (auto *B : Blocks) + B->~Block(); +} + +void LinkGraph::dump(raw_ostream &OS, std::function<StringRef(Edge::Kind)> EdgeKindToName) { if (!EdgeKindToName) EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; - OS << "Defined atoms:\n"; - for (auto *DA : defined_atoms()) { - OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + OS << "Symbols:\n"; + for (auto *Sym : defined_symbols()) { + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; - for (auto &E : DA->edges()) { - OS << " "; - StringRef EdgeName = (E.getKind() < Edge::FirstRelocation - ? getGenericEdgeKindName(E.getKind()) - : EdgeKindToName(E.getKind())); - - if (!EdgeName.empty()) - printEdge(OS, *DA, E, EdgeName); - else { - auto EdgeNumberString = std::to_string(E.getKind()); - printEdge(OS, *DA, E, EdgeNumberString); + if (Sym->isDefined()) { + for (auto &E : Sym->getBlock().edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, Sym->getBlock(), E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, Sym->getBlock(), E, EdgeNumberString); + } + OS << "\n"; } - OS << "\n"; } } - OS << "Absolute atoms:\n"; - for (auto *A : absolute_atoms()) - OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + OS << "Absolute symbols:\n"; + for (auto *Sym : absolute_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; - OS << "External atoms:\n"; - for (auto *A : external_atoms()) - OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + OS << "External symbols:\n"; + for (auto *Sym : external_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; } +void JITLinkAsyncLookupContinuation::anchor() {} + JITLinkContext::~JITLinkContext() {} bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { return true; } -AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { - return AtomGraphPassFunction(); +LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return LinkGraphPassFunction(); } Error JITLinkContext::modifyPassConfig(const Triple &TT, @@ -152,9 +208,9 @@ Error JITLinkContext::modifyPassConfig(const Triple &TT, return Error::success(); } -Error markAllAtomsLive(AtomGraph &G) { - for (auto *DA : G.defined_atoms()) - DA->setLive(true); +Error markAllSymbolsLive(LinkGraph &G) { + for (auto *Sym : G.defined_symbols()) + Sym->setLive(true); return Error::success(); } diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 96e074da122b..d4270b5aa796 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "JITLinkGeneric.h" -#include "EHFrameSupportImpl.h" #include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/MemoryBuffer.h" @@ -25,7 +24,7 @@ JITLinkerBase::~JITLinkerBase() {} void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { - // Build the atom graph. + // Build the link graph. if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) G = std::move(*GraphOrErr); else @@ -33,33 +32,33 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { assert(G && "Graph should have been created by buildGraph above"); // Prune and optimize the graph. - if (auto Err = runPasses(Passes.PrePrunePasses, *G)) + if (auto Err = runPasses(Passes.PrePrunePasses)) return Ctx->notifyFailed(std::move(Err)); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n"; dumpGraph(dbgs()); }); prune(*G); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n"; dumpGraph(dbgs()); }); // Run post-pruning passes. - if (auto Err = runPasses(Passes.PostPrunePasses, *G)) + if (auto Err = runPasses(Passes.PostPrunePasses)) return Ctx->notifyFailed(std::move(Err)); - // Sort atoms into segments. - layOutAtoms(); + // Sort blocks into segments. + auto Layout = layOutBlocks(); // Allocate memory for segments. if (auto Err = allocateSegments(Layout)) return Ctx->notifyFailed(std::move(Err)); - // Notify client that the defined atoms have been assigned addresses. + // Notify client that the defined symbols have been assigned addresses. Ctx->notifyResolved(*G); auto ExternalSymbols = getExternalSymbolNames(); @@ -74,42 +73,42 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { // [Self=std::move(Self)](Expected<AsyncLookupResult> Result) { // Self->linkPhase2(std::move(Self), std::move(Result)); // }); - // - // FIXME: Use move capture once we have c++14. auto *TmpCtx = Ctx.get(); - auto *UnownedSelf = Self.release(); - auto Phase2Continuation = - [UnownedSelf](Expected<AsyncLookupResult> LookupResult) { - std::unique_ptr<JITLinkerBase> Self(UnownedSelf); - UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); - }; - TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation)); + TmpCtx->lookup(std::move(ExternalSymbols), + createLookupContinuation( + [S = std::move(Self), L = std::move(Layout)]( + Expected<AsyncLookupResult> LookupResult) mutable { + auto &TmpSelf = *S; + TmpSelf.linkPhase2(std::move(S), std::move(LookupResult), + std::move(L)); + })); } void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, - Expected<AsyncLookupResult> LR) { + Expected<AsyncLookupResult> LR, + SegmentLayoutMap Layout) { // If the lookup failed, bail out. if (!LR) return deallocateAndBailOut(LR.takeError()); - // Assign addresses to external atoms. + // Assign addresses to external addressables. applyLookupResult(*LR); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; dumpGraph(dbgs()); }); - // Copy atom content to working memory and fix up. - if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + // Copy block content to working memory and fix up. + if (auto Err = copyAndFixUpBlocks(Layout, *Alloc)) return deallocateAndBailOut(std::move(Err)); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n"; dumpGraph(dbgs()); }); - if (auto Err = runPasses(Passes.PostFixupPasses, *G)) + if (auto Err = runPasses(Passes.PostFixupPasses)) return deallocateAndBailOut(std::move(Err)); // FIXME: Use move capture once we have c++14. @@ -128,82 +127,38 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) { Ctx->notifyFinalized(std::move(Alloc)); } -Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) { +Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) { for (auto &P : Passes) - if (auto Err = P(G)) + if (auto Err = P(*G)) return Err; return Error::success(); } -void JITLinkerBase::layOutAtoms() { - // Group sections by protections, and whether or not they're zero-fill. - for (auto &S : G->sections()) { +JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { - // Skip empty sections. - if (S.atoms_empty()) - continue; + SegmentLayoutMap Layout; - auto &SL = Layout[S.getProtectionFlags()]; - if (S.isZeroFill()) - SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S)); + /// Partition blocks based on permissions and content vs. zero-fill. + for (auto *B : G->blocks()) { + auto &SegLists = Layout[B->getSection().getProtectionFlags()]; + if (!B->isZeroFill()) + SegLists.ContentBlocks.push_back(B); else - SL.ContentSections.push_back(SegmentLayout::SectionLayout(S)); + SegLists.ZeroFillBlocks.push_back(B); } - // Sort sections within the layout by ordinal. - { - auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS, - const SegmentLayout::SectionLayout &RHS) { - return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal(); + /// Sort blocks within each list. + for (auto &KV : Layout) { + + auto CompareBlocks = [](const Block *LHS, const Block *RHS) { + if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal()) + return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal(); + return LHS->getOrdinal() < RHS->getOrdinal(); }; - for (auto &KV : Layout) { - auto &SL = KV.second; - std::sort(SL.ContentSections.begin(), SL.ContentSections.end(), - CompareByOrdinal); - std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(), - CompareByOrdinal); - } - } - // Add atoms to the sections. - for (auto &KV : Layout) { - auto &SL = KV.second; - for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) { - for (auto &SI : *SIList) { - // First build the set of layout-heads (i.e. "heads" of layout-next - // chains) by copying the section atoms, then eliminating any that - // appear as layout-next targets. - DenseSet<DefinedAtom *> LayoutHeads; - for (auto *DA : SI.S->atoms()) - LayoutHeads.insert(DA); - - for (auto *DA : SI.S->atoms()) - if (DA->hasLayoutNext()) - LayoutHeads.erase(&DA->getLayoutNext()); - - // Next, sort the layout heads by address order. - std::vector<DefinedAtom *> OrderedLayoutHeads; - OrderedLayoutHeads.reserve(LayoutHeads.size()); - for (auto *DA : LayoutHeads) - OrderedLayoutHeads.push_back(DA); - - // Now sort the list of layout heads by address. - std::sort(OrderedLayoutHeads.begin(), OrderedLayoutHeads.end(), - [](const DefinedAtom *LHS, const DefinedAtom *RHS) { - return LHS->getAddress() < RHS->getAddress(); - }); - - // Now populate the SI.Atoms field by appending each of the chains. - for (auto *DA : OrderedLayoutHeads) { - SI.Atoms.push_back(DA); - while (DA->hasLayoutNext()) { - auto &Next = DA->getLayoutNext(); - SI.Atoms.push_back(&Next); - DA = &Next; - } - } - } - } + auto &SegLists = KV.second; + llvm::sort(SegLists.ContentBlocks, CompareBlocks); + llvm::sort(SegLists.ZeroFillBlocks, CompareBlocks); } LLVM_DEBUG({ @@ -213,18 +168,16 @@ void JITLinkerBase::layOutAtoms() { << static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n"; auto &SL = KV.second; for (auto &SIEntry : - {std::make_pair(&SL.ContentSections, "content sections"), - std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) { - auto &SIList = *SIEntry.first; + {std::make_pair(&SL.ContentBlocks, "content block"), + std::make_pair(&SL.ZeroFillBlocks, "zero-fill block")}) { dbgs() << " " << SIEntry.second << ":\n"; - for (auto &SI : SIList) { - dbgs() << " " << SI.S->getName() << ":\n"; - for (auto *DA : SI.Atoms) - dbgs() << " " << *DA << "\n"; - } + for (auto *B : *SIEntry.first) + dbgs() << " " << *B << "\n"; } } }); + + return Layout; } Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { @@ -234,74 +187,36 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { JITLinkMemoryManager::SegmentsRequestMap Segments; for (auto &KV : Layout) { auto &Prot = KV.first; - auto &SegLayout = KV.second; + auto &SegLists = KV.second; + + uint64_t SegAlign = 1; // Calculate segment content size. size_t SegContentSize = 0; - for (auto &SI : SegLayout.ContentSections) { - assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); - assert(!SI.Atoms.empty() && "Section layouts must not be empty"); - - // Bump to section alignment before processing atoms. - SegContentSize = alignTo(SegContentSize, SI.S->getAlignment()); - - for (auto *DA : SI.Atoms) { - SegContentSize = alignTo(SegContentSize, DA->getAlignment()); - SegContentSize += DA->getSize(); - } + for (auto *B : SegLists.ContentBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegContentSize = alignToBlock(SegContentSize, *B); + SegContentSize += B->getSize(); } - // Get segment content alignment. - unsigned SegContentAlign = 1; - if (!SegLayout.ContentSections.empty()) { - auto &FirstContentSection = SegLayout.ContentSections.front(); - SegContentAlign = - std::max(FirstContentSection.S->getAlignment(), - FirstContentSection.Atoms.front()->getAlignment()); - } + uint64_t SegZeroFillStart = SegContentSize; + uint64_t SegZeroFillEnd = SegZeroFillStart; - // Calculate segment zero-fill size. - uint64_t SegZeroFillSize = 0; - for (auto &SI : SegLayout.ZeroFillSections) { - assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); - assert(!SI.Atoms.empty() && "Section layouts must not be empty"); - - // Bump to section alignment before processing atoms. - SegZeroFillSize = alignTo(SegZeroFillSize, SI.S->getAlignment()); - - for (auto *DA : SI.Atoms) { - SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment()); - SegZeroFillSize += DA->getSize(); - } - } - - // Calculate segment zero-fill alignment. - uint32_t SegZeroFillAlign = 1; - - if (!SegLayout.ZeroFillSections.empty()) { - auto &FirstZeroFillSection = SegLayout.ZeroFillSections.front(); - SegZeroFillAlign = - std::max(FirstZeroFillSection.S->getAlignment(), - FirstZeroFillSection.Atoms.front()->getAlignment()); + for (auto *B : SegLists.ZeroFillBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegZeroFillEnd = alignToBlock(SegZeroFillEnd, *B); + SegZeroFillEnd += B->getSize(); } - if (SegContentSize == 0) - SegContentAlign = SegZeroFillAlign; - - if (SegContentAlign % SegZeroFillAlign != 0) - return make_error<JITLinkError>("First content atom alignment does not " - "accommodate first zero-fill atom " - "alignment"); - - Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize, - SegZeroFillAlign}; + Segments[Prot] = {SegAlign, SegContentSize, + SegZeroFillEnd - SegZeroFillStart}; LLVM_DEBUG({ dbgs() << (&KV == &*Layout.begin() ? "" : "; ") - << static_cast<sys::Memory::ProtectionFlags>(Prot) << ": " - << SegContentSize << " content bytes (alignment " - << SegContentAlign << ") + " << SegZeroFillSize - << " zero-fill bytes (alignment " << SegZeroFillAlign << ")"; + << static_cast<sys::Memory::ProtectionFlags>(Prot) + << ": alignment = " << SegAlign + << ", content size = " << SegContentSize + << ", zero-fill size = " << (SegZeroFillEnd - SegZeroFillStart); }); } LLVM_DEBUG(dbgs() << " }\n"); @@ -320,22 +235,19 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { } }); - // Update atom target addresses. + // Update block target addresses. for (auto &KV : Layout) { auto &Prot = KV.first; auto &SL = KV.second; - JITTargetAddress AtomTargetAddr = + JITTargetAddress NextBlockAddr = Alloc->getTargetMemory(static_cast<sys::Memory::ProtectionFlags>(Prot)); - for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) - for (auto &SI : *SIList) { - AtomTargetAddr = alignTo(AtomTargetAddr, SI.S->getAlignment()); - for (auto *DA : SI.Atoms) { - AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment()); - DA->setAddress(AtomTargetAddr); - AtomTargetAddr += DA->getSize(); - } + for (auto *SIList : {&SL.ContentBlocks, &SL.ZeroFillBlocks}) + for (auto *B : *SIList) { + NextBlockAddr = alignToBlock(NextBlockAddr, *B); + B->setAddress(NextBlockAddr); + NextBlockAddr += B->getSize(); } } @@ -343,34 +255,35 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { } DenseSet<StringRef> JITLinkerBase::getExternalSymbolNames() const { - // Identify unresolved external atoms. + // Identify unresolved external symbols. DenseSet<StringRef> UnresolvedExternals; - for (auto *DA : G->external_atoms()) { - assert(DA->getAddress() == 0 && + for (auto *Sym : G->external_symbols()) { + assert(Sym->getAddress() == 0 && "External has already been assigned an address"); - assert(DA->getName() != StringRef() && DA->getName() != "" && + assert(Sym->getName() != StringRef() && Sym->getName() != "" && "Externals must be named"); - UnresolvedExternals.insert(DA->getName()); + UnresolvedExternals.insert(Sym->getName()); } return UnresolvedExternals; } void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { - for (auto &KV : Result) { - Atom &A = G->getAtomByName(KV.first); - assert(A.getAddress() == 0 && "Atom already resolved"); - A.setAddress(KV.second.getAddress()); + for (auto *Sym : G->external_symbols()) { + assert(Sym->getAddress() == 0 && "Symbol already resolved"); + assert(!Sym->isDefined() && "Symbol being resolved is already defined"); + assert(Result.count(Sym->getName()) && "Missing resolution for symbol"); + Sym->getAddressable().setAddress(Result[Sym->getName()].getAddress()); } LLVM_DEBUG({ dbgs() << "Externals after applying lookup result:\n"; - for (auto *A : G->external_atoms()) - dbgs() << " " << A->getName() << ": " - << formatv("{0:x16}", A->getAddress()) << "\n"; + for (auto *Sym : G->external_symbols()) + dbgs() << " " << Sym->getName() << ": " + << formatv("{0:x16}", Sym->getAddress()) << "\n"; }); - assert(llvm::all_of(G->external_atoms(), - [](Atom *A) { return A->getAddress() != 0; }) && - "All atoms should have been resolved by this point"); + assert(llvm::all_of(G->external_symbols(), + [](Symbol *Sym) { return Sym->getAddress() != 0; }) && + "All symbols should have been resolved by this point"); } void JITLinkerBase::deallocateAndBailOut(Error Err) { @@ -384,96 +297,60 @@ void JITLinkerBase::dumpGraph(raw_ostream &OS) { G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); } -void prune(AtomGraph &G) { - std::vector<DefinedAtom *> Worklist; - DenseMap<DefinedAtom *, std::vector<Edge *>> EdgesToUpdate; - - // Build the initial worklist from all atoms initially live. - for (auto *DA : G.defined_atoms()) { - if (!DA->isLive() || DA->shouldDiscard()) - continue; - - for (auto &E : DA->edges()) { - if (!E.getTarget().isDefined()) - continue; +void prune(LinkGraph &G) { + std::vector<Symbol *> Worklist; + DenseSet<Block *> VisitedBlocks; - auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); + // Build the initial worklist from all symbols initially live. + for (auto *Sym : G.defined_symbols()) + if (Sym->isLive()) + Worklist.push_back(Sym); - if (EDT.shouldDiscard()) - EdgesToUpdate[&EDT].push_back(&E); - else if (E.isKeepAlive() && !EDT.isLive()) - Worklist.push_back(&EDT); - } - } - - // Propagate live flags to all atoms reachable from the initial live set. + // Propagate live flags to all symbols reachable from the initial live set. while (!Worklist.empty()) { - DefinedAtom &NextLive = *Worklist.back(); + auto *Sym = Worklist.back(); Worklist.pop_back(); - assert(!NextLive.shouldDiscard() && - "should-discard nodes should never make it into the worklist"); + auto &B = Sym->getBlock(); - // If this atom has already been marked as live, or is marked to be - // discarded, then skip it. - if (NextLive.isLive()) + // Skip addressables that we've visited before. + if (VisitedBlocks.count(&B)) continue; - // Otherwise set it as live and add any non-live atoms that it points to - // to the worklist. - NextLive.setLive(true); - - for (auto &E : NextLive.edges()) { - if (!E.getTarget().isDefined()) - continue; + VisitedBlocks.insert(&B); - auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); - - if (EDT.shouldDiscard()) - EdgesToUpdate[&EDT].push_back(&E); - else if (E.isKeepAlive() && !EDT.isLive()) - Worklist.push_back(&EDT); + for (auto &E : Sym->getBlock().edges()) { + if (E.getTarget().isDefined() && !E.getTarget().isLive()) { + E.getTarget().setLive(true); + Worklist.push_back(&E.getTarget()); + } } } - // Collect atoms to remove, then remove them from the graph. - std::vector<DefinedAtom *> AtomsToRemove; - for (auto *DA : G.defined_atoms()) - if (DA->shouldDiscard() || !DA->isLive()) - AtomsToRemove.push_back(DA); - - LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); - for (auto *DA : AtomsToRemove) { - LLVM_DEBUG(dbgs() << " " << *DA << "... "); - - // Check whether we need to replace this atom with an external atom. - // - // We replace if all of the following hold: - // (1) The atom is marked should-discard, - // (2) it has live edges (i.e. edges from live atoms) pointing to it. - // - // Otherwise we simply delete the atom. - - G.removeDefinedAtom(*DA); - - auto EdgesToUpdateItr = EdgesToUpdate.find(DA); - if (EdgesToUpdateItr != EdgesToUpdate.end()) { - auto &ExternalReplacement = G.addExternalAtom(DA->getName()); - for (auto *EdgeToUpdate : EdgesToUpdateItr->second) - EdgeToUpdate->setTarget(ExternalReplacement); - LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n"); - } else - LLVM_DEBUG(dbgs() << "deleted\n"); + // Collect all the symbols to remove, then remove them. + { + LLVM_DEBUG(dbgs() << "Dead-stripping symbols:\n"); + std::vector<Symbol *> SymbolsToRemove; + for (auto *Sym : G.defined_symbols()) + if (!Sym->isLive()) + SymbolsToRemove.push_back(Sym); + for (auto *Sym : SymbolsToRemove) { + LLVM_DEBUG(dbgs() << " " << *Sym << "...\n"); + G.removeDefinedSymbol(*Sym); + } } - // Finally, discard any absolute symbols that were marked should-discard. + // Delete any unused blocks. { - std::vector<Atom *> AbsoluteAtomsToRemove; - for (auto *A : G.absolute_atoms()) - if (A->shouldDiscard() || A->isLive()) - AbsoluteAtomsToRemove.push_back(A); - for (auto *A : AbsoluteAtomsToRemove) - G.removeAbsoluteAtom(*A); + LLVM_DEBUG(dbgs() << "Dead-stripping blocks:\n"); + std::vector<Block *> BlocksToRemove; + for (auto *B : G.blocks()) + if (!VisitedBlocks.count(B)) + BlocksToRemove.push_back(B); + for (auto *B : BlocksToRemove) { + LLVM_DEBUG(dbgs() << " " << *B << "...\n"); + G.removeBlock(*B); + } } } diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index e6fd6e38f7a6..07dee6cee200 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -41,39 +41,32 @@ public: protected: struct SegmentLayout { - using SectionAtomsList = std::vector<DefinedAtom *>; - struct SectionLayout { - SectionLayout(Section &S) : S(&S) {} + using BlocksList = std::vector<Block *>; - Section *S; - SectionAtomsList Atoms; - }; - - using SectionLayoutList = std::vector<SectionLayout>; - - SectionLayoutList ContentSections; - SectionLayoutList ZeroFillSections; + BlocksList ContentBlocks; + BlocksList ZeroFillBlocks; }; using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>; // Phase 1: - // 1.1: Build atom graph + // 1.1: Build link graph // 1.2: Run pre-prune passes // 1.2: Prune graph // 1.3: Run post-prune passes - // 1.4: Sort atoms into segments + // 1.4: Sort blocks into segments // 1.5: Allocate segment memory // 1.6: Identify externals and make an async call to resolve function void linkPhase1(std::unique_ptr<JITLinkerBase> Self); // Phase 2: // 2.1: Apply resolution results - // 2.2: Fix up atom contents + // 2.2: Fix up block contents // 2.3: Call OnResolved callback // 2.3: Make an async call to transfer and finalize memory. void linkPhase2(std::unique_ptr<JITLinkerBase> Self, - Expected<AsyncLookupResult> LookupResult); + Expected<AsyncLookupResult> LookupResult, + SegmentLayoutMap Layout); // Phase 3: // 3.1: Call OnFinalized callback, handing off allocation. @@ -81,24 +74,37 @@ protected: // Build a graph from the given object buffer. // To be implemented by the client. - virtual Expected<std::unique_ptr<AtomGraph>> + virtual Expected<std::unique_ptr<LinkGraph>> buildGraph(MemoryBufferRef ObjBuffer) = 0; - // For debug dumping of the atom graph. + // For debug dumping of the link graph. virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + // Alight a JITTargetAddress to conform with block alignment requirements. + static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { + uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); + return Addr + Delta; + } + + // Alight a pointer to conform with block alignment requirements. + static char *alignToBlock(char *P, Block &B) { + uint64_t PAddr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(P)); + uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment(); + return P + Delta; + } + private: // Run all passes in the given pass list, bailing out immediately if any pass // returns an error. - Error runPasses(AtomGraphPassList &Passes, AtomGraph &G); + Error runPasses(LinkGraphPassList &Passes); - // Copy atom contents and apply relocations. + // Copy block contents and apply relocations. // Implemented in JITLinker. virtual Error - copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const = 0; + copyAndFixUpBlocks(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; - void layOutAtoms(); + SegmentLayoutMap layOutBlocks(); Error allocateSegments(const SegmentLayoutMap &Layout); DenseSet<StringRef> getExternalSymbolNames() const; void applyLookupResult(AsyncLookupResult LR); @@ -108,8 +114,7 @@ private: std::unique_ptr<JITLinkContext> Ctx; PassConfiguration Passes; - std::unique_ptr<AtomGraph> G; - SegmentLayoutMap Layout; + std::unique_ptr<LinkGraph> G; std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc; }; @@ -121,7 +126,7 @@ public: /// Link should be called with the constructor arguments for LinkerImpl, which /// will be forwarded to the constructor. template <typename... ArgTs> static void link(ArgTs &&... Args) { - auto L = llvm::make_unique<LinkerImpl>(std::forward<ArgTs>(Args)...); + auto L = std::make_unique<LinkerImpl>(std::forward<ArgTs>(Args)...); // Ownership of the linker is passed into the linker's doLink function to // allow it to be passed on to async continuations. @@ -140,17 +145,17 @@ private: } Error - copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const override { - LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + copyAndFixUpBlocks(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n"); for (auto &KV : Layout) { auto &Prot = KV.first; auto &SegLayout = KV.second; auto SegMem = Alloc.getWorkingMemory( static_cast<sys::Memory::ProtectionFlags>(Prot)); - char *LastAtomEnd = SegMem.data(); - char *AtomDataPtr = LastAtomEnd; + char *LastBlockEnd = SegMem.data(); + char *BlockDataPtr = LastBlockEnd; LLVM_DEBUG({ dbgs() << " Processing segment " @@ -160,93 +165,79 @@ private: << " ]\n Processing content sections:\n"; }); - for (auto &SI : SegLayout.ContentSections) { - LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n"); + for (auto *B : SegLayout.ContentBlocks) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + + // Pad to alignment/alignment-offset. + BlockDataPtr = alignToBlock(BlockDataPtr, *B); - AtomDataPtr += alignmentAdjustment(AtomDataPtr, SI.S->getAlignment()); + LLVM_DEBUG({ + dbgs() << " Bumped block pointer to " + << (const void *)BlockDataPtr << " to meet block alignment " + << B->getAlignment() << " and alignment offset " + << B->getAlignmentOffset() << "\n"; + }); + // Zero pad up to alignment. LLVM_DEBUG({ - dbgs() << " Bumped atom pointer to " << (const void *)AtomDataPtr - << " to meet section alignment " - << " of " << SI.S->getAlignment() << "\n"; + if (LastBlockEnd != BlockDataPtr) + dbgs() << " Zero padding from " << (const void *)LastBlockEnd + << " to " << (const void *)BlockDataPtr << "\n"; }); - for (auto *DA : SI.Atoms) { - - // Align. - AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment()); - LLVM_DEBUG({ - dbgs() << " Bumped atom pointer to " - << (const void *)AtomDataPtr << " to meet alignment of " - << DA->getAlignment() << "\n"; - }); - - // Zero pad up to alignment. - LLVM_DEBUG({ - if (LastAtomEnd != AtomDataPtr) - dbgs() << " Zero padding from " << (const void *)LastAtomEnd - << " to " << (const void *)AtomDataPtr << "\n"; - }); - while (LastAtomEnd != AtomDataPtr) - *LastAtomEnd++ = 0; - - // Copy initial atom content. - LLVM_DEBUG({ - dbgs() << " Copying atom " << *DA << " content, " - << DA->getContent().size() << " bytes, from " - << (const void *)DA->getContent().data() << " to " - << (const void *)AtomDataPtr << "\n"; - }); - memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size()); - - // Copy atom data and apply fixups. - LLVM_DEBUG(dbgs() << " Applying fixups.\n"); - for (auto &E : DA->edges()) { - - // Skip non-relocation edges. - if (!E.isRelocation()) - continue; - - // Dispatch to LinkerImpl for fixup. - if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr)) - return Err; - } - - // Point the atom's content to the fixed up buffer. - DA->setContent(StringRef(AtomDataPtr, DA->getContent().size())); - - // Update atom end pointer. - LastAtomEnd = AtomDataPtr + DA->getContent().size(); - AtomDataPtr = LastAtomEnd; + while (LastBlockEnd != BlockDataPtr) + *LastBlockEnd++ = 0; + + // Copy initial block content. + LLVM_DEBUG({ + dbgs() << " Copying block " << *B << " content, " + << B->getContent().size() << " bytes, from " + << (const void *)B->getContent().data() << " to " + << (const void *)BlockDataPtr << "\n"; + }); + memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); + + // Copy Block data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : B->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*B, E, BlockDataPtr)) + return Err; } + + // Point the block's content to the fixed up buffer. + B->setContent(StringRef(BlockDataPtr, B->getContent().size())); + + // Update block end pointer. + LastBlockEnd = BlockDataPtr + B->getContent().size(); + BlockDataPtr = LastBlockEnd; } // Zero pad the rest of the segment. LLVM_DEBUG({ dbgs() << " Zero padding end of segment from " - << (const void *)LastAtomEnd << " to " + << (const void *)LastBlockEnd << " to " << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; }); - while (LastAtomEnd != SegMem.data() + SegMem.size()) - *LastAtomEnd++ = 0; + while (LastBlockEnd != SegMem.data() + SegMem.size()) + *LastBlockEnd++ = 0; } return Error::success(); } }; -/// Dead strips and replaces discarded definitions with external atoms. +/// Removes dead symbols/blocks/addressables. /// -/// Finds the set of nodes reachable from any node initially marked live -/// (nodes marked should-discard are treated as not live, even if they are -/// reachable). All nodes not marked as live at the end of this process, -/// are deleted. Nodes that are live, but marked should-discard are replaced -/// with external atoms and all edges to them are re-written. -void prune(AtomGraph &G); - -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); +/// Finds the set of symbols and addressables reachable from any symbol +/// initially marked live. All symbols/addressables not marked live at the end +/// of this process are removed. +void prune(LinkGraph &G); } // end namespace jitlink } // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index 267307cfde05..9e0d207e8bdb 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -38,9 +38,21 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { OnFinalize(applyProtections()); } Error deallocate() override { - for (auto &KV : SegBlocks) - if (auto EC = sys::Memory::releaseMappedMemory(KV.second)) - return errorCodeToError(EC); + if (SegBlocks.empty()) + return Error::success(); + void *SlabStart = SegBlocks.begin()->second.base(); + char *SlabEnd = (char *)SlabStart; + for (auto &KV : SegBlocks) { + SlabStart = std::min(SlabStart, KV.second.base()); + SlabEnd = std::max(SlabEnd, (char *)(KV.second.base()) + + KV.second.allocatedSize()); + } + size_t SlabSize = SlabEnd - (char *)SlabStart; + assert((SlabSize % sys::Process::getPageSizeEstimate()) == 0 && + "Slab size is not a multiple of page size"); + sys::MemoryBlock Slab(SlabStart, SlabSize); + if (auto EC = sys::Memory::releaseMappedMemory(Slab)) + return errorCodeToError(EC); return Error::success(); } @@ -61,37 +73,52 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { AllocationMap SegBlocks; }; + if (!isPowerOf2_64((uint64_t)sys::Process::getPageSizeEstimate())) + return make_error<StringError>("Page size is not a power of 2", + inconvertibleErrorCode()); + AllocationMap Blocks; const sys::Memory::ProtectionFlags ReadWrite = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | sys::Memory::MF_WRITE); + // Compute the total number of pages to allocate. + size_t TotalSize = 0; for (auto &KV : Request) { - auto &Seg = KV.second; + const auto &Seg = KV.second; - if (Seg.getContentAlignment() > sys::Process::getPageSizeEstimate()) + if (Seg.getAlignment() > sys::Process::getPageSizeEstimate()) return make_error<StringError>("Cannot request higher than page " "alignment", inconvertibleErrorCode()); - if (sys::Process::getPageSizeEstimate() % Seg.getContentAlignment() != 0) - return make_error<StringError>("Page size is not a multiple of " - "alignment", - inconvertibleErrorCode()); + TotalSize = alignTo(TotalSize, sys::Process::getPageSizeEstimate()); + TotalSize += Seg.getContentSize(); + TotalSize += Seg.getZeroFillSize(); + } + + // Allocate one slab to cover all the segments. + std::error_code EC; + auto SlabRemaining = + sys::Memory::allocateMappedMemory(TotalSize, nullptr, ReadWrite, EC); + + if (EC) + return errorCodeToError(EC); + + // Allocate segment memory from the slab. + for (auto &KV : Request) { - uint64_t ZeroFillStart = - alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); - uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); + const auto &Seg = KV.second; - std::error_code EC; - auto SegMem = - sys::Memory::allocateMappedMemory(SegmentSize, nullptr, ReadWrite, EC); + uint64_t SegmentSize = alignTo(Seg.getContentSize() + Seg.getZeroFillSize(), + sys::Process::getPageSizeEstimate()); - if (EC) - return errorCodeToError(EC); + sys::MemoryBlock SegMem(SlabRemaining.base(), SegmentSize); + SlabRemaining = sys::MemoryBlock((char *)SlabRemaining.base() + SegmentSize, + SegmentSize); // Zero out the zero-fill memory. - memset(static_cast<char *>(SegMem.base()) + ZeroFillStart, 0, + memset(static_cast<char *>(SegMem.base()) + Seg.getContentSize(), 0, Seg.getZeroFillSize()); // Record the block for this segment. diff --git a/lib/ExecutionEngine/JITLink/MachO.cpp b/lib/ExecutionEngine/JITLink/MachO.cpp index 15995b8ce98f..58bc0f56e155 100644 --- a/lib/ExecutionEngine/JITLink/MachO.cpp +++ b/lib/ExecutionEngine/JITLink/MachO.cpp @@ -14,6 +14,7 @@ #include "llvm/ExecutionEngine/JITLink/MachO.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Format.h" @@ -64,6 +65,8 @@ void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) { }); switch (Header.cputype) { + case MachO::CPU_TYPE_ARM64: + return jitLink_MachO_arm64(std::move(Ctx)); case MachO::CPU_TYPE_X86_64: return jitLink_MachO_x86_64(std::move(Ctx)); } diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp deleted file mode 100644 index 1501c7ad0bc5..000000000000 --- a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp +++ /dev/null @@ -1,411 +0,0 @@ -//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Generic MachO AtomGraph buliding code. -// -//===----------------------------------------------------------------------===// - -#include "MachOAtomGraphBuilder.h" - -#define DEBUG_TYPE "jitlink" - -namespace llvm { -namespace jitlink { - -MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} - -Expected<std::unique_ptr<AtomGraph>> MachOAtomGraphBuilder::buildGraph() { - if (auto Err = parseSections()) - return std::move(Err); - - if (auto Err = addAtoms()) - return std::move(Err); - - if (auto Err = addRelocations()) - return std::move(Err); - - return std::move(G); -} - -MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) - : Obj(Obj), - G(llvm::make_unique<AtomGraph>(Obj.getFileName(), getPointerSize(Obj), - getEndianness(Obj))) {} - -void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, - CustomAtomizeFunction Atomizer) { - assert(!CustomAtomizeFunctions.count(SectionName) && - "Custom atomizer for this section already exists"); - CustomAtomizeFunctions[SectionName] = std::move(Atomizer); -} - -bool MachOAtomGraphBuilder::areLayoutLocked(const Atom &A, const Atom &B) { - // If these atoms are the same then they're trivially "locked". - if (&A == &B) - return true; - - // If A and B are different, check whether either is undefined. (in which - // case they are not locked). - if (!A.isDefined() || !B.isDefined()) - return false; - - // A and B are different, but they're both defined atoms. We need to check - // whether they're part of the same alt_entry chain. - auto &DA = static_cast<const DefinedAtom &>(A); - auto &DB = static_cast<const DefinedAtom &>(B); - - auto AStartItr = AltEntryStarts.find(&DA); - if (AStartItr == AltEntryStarts.end()) // If A is not in a chain bail out. - return false; - - auto BStartItr = AltEntryStarts.find(&DB); - if (BStartItr == AltEntryStarts.end()) // If B is not in a chain bail out. - return false; - - // A and B are layout locked if they're in the same chain. - return AStartItr->second == BStartItr->second; -} - -unsigned -MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { - return Obj.is64Bit() ? 8 : 4; -} - -support::endianness -MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { - return Obj.isLittleEndian() ? support::little : support::big; -} - -MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() { - if (!CommonSymbolsSection) { - auto Prot = static_cast<sys::Memory::ProtectionFlags>( - sys::Memory::MF_READ | sys::Memory::MF_WRITE); - auto &GenericSection = G->createSection("<common>", 1, Prot, true); - CommonSymbolsSection = MachOSection(GenericSection); - } - return *CommonSymbolsSection; -} - -Error MachOAtomGraphBuilder::parseSections() { - for (auto &SecRef : Obj.sections()) { - assert((SecRef.getAlignment() <= std::numeric_limits<uint32_t>::max()) && - "Section alignment does not fit in 32 bits"); - - StringRef Name; - if (auto EC = SecRef.getName(Name)) - return errorCodeToError(EC); - - unsigned SectionIndex = SecRef.getIndex() + 1; - - uint32_t Align = SecRef.getAlignment(); - if (!isPowerOf2_32(Align)) - return make_error<JITLinkError>("Section " + Name + - " has non-power-of-2 " - "alignment"); - - // FIXME: Get real section permissions - // How, exactly, on MachO? - sys::Memory::ProtectionFlags Prot; - if (SecRef.isText()) - Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - else - Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_WRITE); - - auto &GenericSection = G->createSection(Name, Align, Prot, SecRef.isBSS()); - - LLVM_DEBUG({ - dbgs() << "Adding section " << Name << ": " - << format("0x%016" PRIx64, SecRef.getAddress()) - << ", align: " << SecRef.getAlignment() << "\n"; - }); - - assert(!Sections.count(SectionIndex) && "Section index already in use"); - - auto &MachOSec = - Sections - .try_emplace(SectionIndex, GenericSection, SecRef.getAddress(), - SecRef.getAlignment()) - .first->second; - - if (!SecRef.isVirtual()) { - // If this section has content then record it. - Expected<StringRef> Content = SecRef.getContents(); - if (!Content) - return Content.takeError(); - if (Content->size() != SecRef.getSize()) - return make_error<JITLinkError>("Section content size does not match " - "declared size for " + - Name); - MachOSec.setContent(*Content); - } else { - // If this is a zero-fill section then just record the size. - MachOSec.setZeroFill(SecRef.getSize()); - } - - uint32_t SectionFlags = - Obj.is64Bit() ? Obj.getSection64(SecRef.getRawDataRefImpl()).flags - : Obj.getSection(SecRef.getRawDataRefImpl()).flags; - - MachOSec.setNoDeadStrip(SectionFlags & MachO::S_ATTR_NO_DEAD_STRIP); - } - - return Error::success(); -} - -// Adds atoms with identified start addresses (but not lengths) for all named -// atoms. -// Also, for every section that contains named atoms, but does not have an -// atom at offset zero of that section, constructs an anonymous atom covering -// that range. -Error MachOAtomGraphBuilder::addNonCustomAtoms() { - using AddrToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; - DenseMap<MachOSection *, AddrToAtomMap> SecToAtoms; - - DenseMap<MachOSection *, unsigned> FirstOrdinal; - std::vector<DefinedAtom *> AltEntryAtoms; - - DenseSet<StringRef> ProcessedSymbols; // Used to check for duplicate defs. - - for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; - ++SymI) { - object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); - - auto Name = Sym.getName(); - if (!Name) - return Name.takeError(); - - // Bail out on duplicate definitions: There should never be more than one - // definition for a symbol in a given object file. - if (ProcessedSymbols.count(*Name)) - return make_error<JITLinkError>("Duplicate definition within object: " + - *Name); - else - ProcessedSymbols.insert(*Name); - - auto Addr = Sym.getAddress(); - if (!Addr) - return Addr.takeError(); - - auto SymType = Sym.getType(); - if (!SymType) - return SymType.takeError(); - - auto Flags = Sym.getFlags(); - - if (Flags & object::SymbolRef::SF_Undefined) { - LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); - G->addExternalAtom(*Name); - continue; - } else if (Flags & object::SymbolRef::SF_Absolute) { - LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " - << format("0x%016" PRIx64, *Addr) << "\n"); - auto &A = G->addAbsoluteAtom(*Name, *Addr); - A.setGlobal(Flags & object::SymbolRef::SF_Global); - A.setExported(Flags & object::SymbolRef::SF_Exported); - A.setWeak(Flags & object::SymbolRef::SF_Weak); - continue; - } else if (Flags & object::SymbolRef::SF_Common) { - LLVM_DEBUG({ - dbgs() << "Adding common \"" << *Name - << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n"; - }); - auto &A = - G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr, - std::max(Sym.getAlignment(), 1U), - Obj.getCommonSymbolSize(Sym.getRawDataRefImpl())); - A.setGlobal(Flags & object::SymbolRef::SF_Global); - A.setExported(Flags & object::SymbolRef::SF_Exported); - continue; - } - - LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); - - // This atom is neither undefined nor absolute, so it must be defined in - // this object. Get its section index. - auto SecItr = Sym.getSection(); - if (!SecItr) - return SecItr.takeError(); - - uint64_t SectionIndex = (*SecItr)->getIndex() + 1; - - LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n"); - - auto SecByIndexItr = Sections.find(SectionIndex); - if (SecByIndexItr == Sections.end()) - return make_error<JITLinkError>("Unrecognized section index in macho"); - - auto &Sec = SecByIndexItr->second; - - auto &DA = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, - std::max(Sym.getAlignment(), 1U)); - - DA.setGlobal(Flags & object::SymbolRef::SF_Global); - DA.setExported(Flags & object::SymbolRef::SF_Exported); - DA.setWeak(Flags & object::SymbolRef::SF_Weak); - - DA.setCallable(*SymType & object::SymbolRef::ST_Function); - - // Check NDesc flags. - { - uint16_t NDesc = 0; - if (Obj.is64Bit()) - NDesc = Obj.getSymbol64TableEntry(SymI->getRawDataRefImpl()).n_desc; - else - NDesc = Obj.getSymbolTableEntry(SymI->getRawDataRefImpl()).n_desc; - - // Record atom for alt-entry post-processing (where the layout-next - // constraints will be added). - if (NDesc & MachO::N_ALT_ENTRY) - AltEntryAtoms.push_back(&DA); - - // If this atom has a no-dead-strip attr attached then mark it live. - if (NDesc & MachO::N_NO_DEAD_STRIP) - DA.setLive(true); - } - - LLVM_DEBUG({ - dbgs() << " Added " << *Name - << " addr: " << format("0x%016" PRIx64, *Addr) - << ", align: " << DA.getAlignment() - << ", section: " << Sec.getGenericSection().getName() << "\n"; - }); - - auto &SecAtoms = SecToAtoms[&Sec]; - SecAtoms[DA.getAddress() - Sec.getAddress()] = &DA; - } - - // Add anonymous atoms. - for (auto &KV : Sections) { - auto &S = KV.second; - - // Skip empty sections. - if (S.empty()) - continue; - - // Skip sections with custom handling. - if (CustomAtomizeFunctions.count(S.getName())) - continue; - - auto SAI = SecToAtoms.find(&S); - - // If S is not in the SecToAtoms map then it contained no named atom. Add - // one anonymous atom to cover the whole section. - if (SAI == SecToAtoms.end()) { - SecToAtoms[&S][0] = &G->addAnonymousAtom( - S.getGenericSection(), S.getAddress(), S.getAlignment()); - continue; - } - - // Otherwise, check whether this section had an atom covering offset zero. - // If not, add one. - auto &SecAtoms = SAI->second; - if (!SecAtoms.count(0)) - SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), - S.getAlignment()); - } - - LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); - - // Set atom contents and any section-based flags. - for (auto &KV : SecToAtoms) { - auto &S = *KV.first; - auto &SecAtoms = KV.second; - - // Iterate the atoms in reverse order and set up their contents. - JITTargetAddress LastAtomAddr = S.getSize(); - for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { - auto Offset = I->first; - auto &A = *I->second; - LLVM_DEBUG({ - dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. " - << S.getAddress() + LastAtomAddr << " ]\n"; - }); - - if (S.isZeroFill()) - A.setZeroFill(LastAtomAddr - Offset); - else - A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); - - // If the section has no-dead-strip set then mark the atom as live. - if (S.isNoDeadStrip()) - A.setLive(true); - - LastAtomAddr = Offset; - } - } - - LLVM_DEBUG(dbgs() << "Adding alt-entry starts\n"); - - // Sort alt-entry atoms by address in ascending order. - llvm::sort(AltEntryAtoms.begin(), AltEntryAtoms.end(), - [](const DefinedAtom *LHS, const DefinedAtom *RHS) { - return LHS->getAddress() < RHS->getAddress(); - }); - - // Process alt-entry atoms in address order to build the table of alt-entry - // atoms to alt-entry chain starts. - for (auto *DA : AltEntryAtoms) { - assert(!AltEntryStarts.count(DA) && "Duplicate entry in AltEntryStarts"); - - // DA is an alt-entry atom. Look for the predecessor atom that it is locked - // to, bailing out if we do not find one. - auto AltEntryPred = G->findAtomByAddress(DA->getAddress() - 1); - if (!AltEntryPred) - return AltEntryPred.takeError(); - - // Add a LayoutNext edge from the predecessor to this atom. - AltEntryPred->setLayoutNext(*DA); - - // Check to see whether the predecessor itself is an alt-entry atom. - auto AltEntryStartItr = AltEntryStarts.find(&*AltEntryPred); - if (AltEntryStartItr != AltEntryStarts.end()) { - // If the predecessor was an alt-entry atom then re-use its value. - LLVM_DEBUG({ - dbgs() << " " << *DA << " -> " << *AltEntryStartItr->second - << " (based on existing entry for " << *AltEntryPred << ")\n"; - }); - AltEntryStarts[DA] = AltEntryStartItr->second; - } else { - // If the predecessor does not have an entry then add an entry for this - // atom (i.e. the alt_entry atom) and a self-reference entry for the - /// predecessory atom that is the start of this chain. - LLVM_DEBUG({ - dbgs() << " " << *AltEntryPred << " -> " << *AltEntryPred << "\n" - << " " << *DA << " -> " << *AltEntryPred << "\n"; - }); - AltEntryStarts[&*AltEntryPred] = &*AltEntryPred; - AltEntryStarts[DA] = &*AltEntryPred; - } - } - - return Error::success(); -} - -Error MachOAtomGraphBuilder::addAtoms() { - // Add all named atoms. - if (auto Err = addNonCustomAtoms()) - return Err; - - // Process special sections. - for (auto &KV : Sections) { - auto &S = KV.second; - auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); - if (HI != CustomAtomizeFunctions.end()) { - auto &Atomize = HI->second; - if (auto Err = Atomize(S)) - return Err; - } - } - - return Error::success(); -} - -} // end namespace jitlink -} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h deleted file mode 100644 index 72d441b24d06..000000000000 --- a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h +++ /dev/null @@ -1,138 +0,0 @@ -//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Generic MachO AtomGraph building code. -// -//===----------------------------------------------------------------------===// - -#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H -#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H - -#include "llvm/ExecutionEngine/JITLink/JITLink.h" - -#include "JITLinkGeneric.h" - -#include "llvm/Object/MachO.h" - -namespace llvm { -namespace jitlink { - -class MachOAtomGraphBuilder { -public: - virtual ~MachOAtomGraphBuilder(); - Expected<std::unique_ptr<AtomGraph>> buildGraph(); - -protected: - using OffsetToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; - - class MachOSection { - public: - MachOSection() = default; - - /// Create a MachO section with the given address and alignment. - MachOSection(Section &GenericSection, JITTargetAddress Address, - unsigned Alignment) - : Address(Address), GenericSection(&GenericSection), - Alignment(Alignment) {} - - /// Create a section without address, content or size (used for common - /// symbol sections). - MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {} - - Section &getGenericSection() const { - assert(GenericSection && "Section is null"); - return *GenericSection; - } - - StringRef getName() const { - assert(GenericSection && "No generic section attached"); - return GenericSection->getName(); - } - - MachOSection &setContent(StringRef Content) { - assert(!ContentPtr && !Size && "Content/zeroFill already set"); - ContentPtr = Content.data(); - Size = Content.size(); - return *this; - } - - MachOSection &setZeroFill(uint64_t Size) { - assert(!ContentPtr && !this->Size && "Content/zeroFill already set"); - this->Size = Size; - return *this; - } - - bool isZeroFill() const { return !ContentPtr; } - - bool empty() const { return getSize() == 0; } - - size_t getSize() const { return Size; } - - StringRef getContent() const { - assert(ContentPtr && "getContent() called on zero-fill section"); - return {ContentPtr, static_cast<size_t>(Size)}; - } - - JITTargetAddress getAddress() const { return Address; } - - unsigned getAlignment() const { return Alignment; } - - MachOSection &setNoDeadStrip(bool NoDeadStrip) { - this->NoDeadStrip = NoDeadStrip; - return *this; - } - - bool isNoDeadStrip() const { return NoDeadStrip; } - - private: - JITTargetAddress Address = 0; - Section *GenericSection = nullptr; - const char *ContentPtr = nullptr; - uint64_t Size = 0; - unsigned Alignment = 0; - bool NoDeadStrip = false; - }; - - using CustomAtomizeFunction = std::function<Error(MachOSection &S)>; - - MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); - - AtomGraph &getGraph() const { return *G; } - - const object::MachOObjectFile &getObject() const { return Obj; } - - void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); - - virtual Error addRelocations() = 0; - - /// Returns true if Atom A and Atom B are at a fixed offset from one another - /// (i.e. if they're part of the same alt-entry chain). - bool areLayoutLocked(const Atom &A, const Atom &B); - -private: - static unsigned getPointerSize(const object::MachOObjectFile &Obj); - static support::endianness getEndianness(const object::MachOObjectFile &Obj); - - MachOSection &getCommonSection(); - - Error parseSections(); - Error addNonCustomAtoms(); - Error addAtoms(); - - const object::MachOObjectFile &Obj; - std::unique_ptr<AtomGraph> G; - DenseMap<const DefinedAtom *, const DefinedAtom *> AltEntryStarts; - DenseMap<unsigned, MachOSection> Sections; - StringMap<CustomAtomizeFunction> CustomAtomizeFunctions; - Optional<MachOSection> CommonSymbolsSection; -}; - -} // end namespace jitlink -} // end namespace llvm - -#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H diff --git a/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp new file mode 100644 index 000000000000..7366f53ebf36 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -0,0 +1,535 @@ +//=--------- MachOLinkGraphBuilder.cpp - MachO LinkGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +static const char *CommonSectionName = "__common"; + +namespace llvm { +namespace jitlink { + +MachOLinkGraphBuilder::~MachOLinkGraphBuilder() {} + +Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() { + + // Sanity check: we only operate on relocatable objects. + if (!Obj.isRelocatableObject()) + return make_error<JITLinkError>("Object is not a relocatable MachO"); + + if (auto Err = createNormalizedSections()) + return std::move(Err); + + if (auto Err = createNormalizedSymbols()) + return std::move(Err); + + if (auto Err = graphifyRegularSymbols()) + return std::move(Err); + + if (auto Err = graphifySectionsWithCustomParsers()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(std::make_unique<LinkGraph>(Obj.getFileName(), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOLinkGraphBuilder::addCustomSectionParser( + StringRef SectionName, SectionParserFunction Parser) { + assert(!CustomSectionParserFunctions.count(SectionName) && + "Custom parser for this section already exists"); + CustomSectionParserFunctions[SectionName] = std::move(Parser); +} + +Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) { + if ((Desc & MachO::N_WEAK_DEF) || (Desc & MachO::N_WEAK_REF)) + return Linkage::Weak; + return Linkage::Strong; +} + +Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) { + if (Name.startswith("l")) + return Scope::Local; + if (Type & MachO::N_PEXT) + return Scope::Hidden; + if (Type & MachO::N_EXT) + return Scope::Default; + return Scope::Local; +} + +bool MachOLinkGraphBuilder::isAltEntry(const NormalizedSymbol &NSym) { + return NSym.Desc & MachO::N_ALT_ENTRY; +} + +unsigned +MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +Section &MachOLinkGraphBuilder::getCommonSection() { + if (!CommonSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + CommonSection = &G->createSection(CommonSectionName, Prot); + } + return *CommonSection; +} + +Error MachOLinkGraphBuilder::createNormalizedSections() { + // Build normalized sections. Verifies that section data is in-range (for + // sections with content) and that address ranges are non-overlapping. + + LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); + + for (auto &SecRef : Obj.sections()) { + NormalizedSection NSec; + uint32_t DataOffset = 0; + + auto SecIndex = Obj.getSectionIndex(SecRef.getRawDataRefImpl()); + + auto Name = SecRef.getName(); + if (!Name) + return Name.takeError(); + + if (Obj.is64Bit()) { + const MachO::section_64 &Sec64 = + Obj.getSection64(SecRef.getRawDataRefImpl()); + + NSec.Address = Sec64.addr; + NSec.Size = Sec64.size; + NSec.Alignment = 1ULL << Sec64.align; + NSec.Flags = Sec64.flags; + DataOffset = Sec64.offset; + } else { + const MachO::section &Sec32 = Obj.getSection(SecRef.getRawDataRefImpl()); + NSec.Address = Sec32.addr; + NSec.Size = Sec32.size; + NSec.Alignment = 1ULL << Sec32.align; + NSec.Flags = Sec32.flags; + DataOffset = Sec32.offset; + } + + LLVM_DEBUG({ + dbgs() << " " << *Name << ": " << formatv("{0:x16}", NSec.Address) + << " -- " << formatv("{0:x16}", NSec.Address + NSec.Size) + << ", align: " << NSec.Alignment << ", index: " << SecIndex + << "\n"; + }); + + // Get the section data if any. + { + unsigned SectionType = NSec.Flags & MachO::SECTION_TYPE; + if (SectionType != MachO::S_ZEROFILL && + SectionType != MachO::S_GB_ZEROFILL) { + + if (DataOffset + NSec.Size > Obj.getData().size()) + return make_error<JITLinkError>( + "Section data extends past end of file"); + + NSec.Data = Obj.getData().data() + DataOffset; + } + } + + // Get prot flags. + // FIXME: Make sure this test is correct (it's probably missing cases + // as-is). + sys::Memory::ProtectionFlags Prot; + if (NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS) + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + NSec.GraphSection = &G->createSection(*Name, Prot); + IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec))); + } + + std::vector<NormalizedSection *> Sections; + Sections.reserve(IndexToSection.size()); + for (auto &KV : IndexToSection) + Sections.push_back(&KV.second); + + // If we didn't end up creating any sections then bail out. The code below + // assumes that we have at least one section. + if (Sections.empty()) + return Error::success(); + + llvm::sort(Sections, + [](const NormalizedSection *LHS, const NormalizedSection *RHS) { + assert(LHS && RHS && "Null section?"); + return LHS->Address < RHS->Address; + }); + + for (unsigned I = 0, E = Sections.size() - 1; I != E; ++I) { + auto &Cur = *Sections[I]; + auto &Next = *Sections[I + 1]; + if (Next.Address < Cur.Address + Cur.Size) + return make_error<JITLinkError>( + "Address range for section " + Cur.GraphSection->getName() + + formatv(" [ {0:x16} -- {1:x16} ] ", Cur.Address, + Cur.Address + Cur.Size) + + "overlaps " + + formatv(" [ {0:x16} -- {1:x16} ] ", Next.Address, + Next.Address + Next.Size)); + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::createNormalizedSymbols() { + LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); + + for (auto &SymRef : Obj.symbols()) { + + unsigned SymbolIndex = Obj.getSymbolIndex(SymRef.getRawDataRefImpl()); + uint64_t Value; + uint32_t NStrX; + uint8_t Type; + uint8_t Sect; + uint16_t Desc; + + if (Obj.is64Bit()) { + const MachO::nlist_64 &NL64 = + Obj.getSymbol64TableEntry(SymRef.getRawDataRefImpl()); + Value = NL64.n_value; + NStrX = NL64.n_strx; + Type = NL64.n_type; + Sect = NL64.n_sect; + Desc = NL64.n_desc; + } else { + const MachO::nlist &NL32 = + Obj.getSymbolTableEntry(SymRef.getRawDataRefImpl()); + Value = NL32.n_value; + NStrX = NL32.n_strx; + Type = NL32.n_type; + Sect = NL32.n_sect; + Desc = NL32.n_desc; + } + + // Skip stabs. + // FIXME: Are there other symbols we should be skipping? + if (Type & MachO::N_STAB) + continue; + + Optional<StringRef> Name; + if (NStrX) { + if (auto NameOrErr = SymRef.getName()) + Name = *NameOrErr; + else + return NameOrErr.takeError(); + } + + LLVM_DEBUG({ + dbgs() << " "; + if (!Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << *Name; + dbgs() << ": value = " << formatv("{0:x16}", Value) + << ", type = " << formatv("{0:x2}", Type) + << ", desc = " << formatv("{0:x4}", Desc) << ", sect = "; + if (Sect) + dbgs() << static_cast<unsigned>(Sect - 1); + else + dbgs() << "none"; + dbgs() << "\n"; + }); + + // If this symbol has a section, sanity check that the addresses line up. + NormalizedSection *NSec = nullptr; + if (Sect != 0) { + if (auto NSecOrErr = findSectionByIndex(Sect - 1)) + NSec = &*NSecOrErr; + else + return NSecOrErr.takeError(); + + if (Value < NSec->Address || Value > NSec->Address + NSec->Size) + return make_error<JITLinkError>("Symbol address does not fall within " + "section"); + } + + IndexToSymbol[SymbolIndex] = + &createNormalizedSymbol(*Name, Value, Type, Sect, Desc, + getLinkage(Type), getScope(*Name, Type)); + } + + return Error::success(); +} + +void MachOLinkGraphBuilder::addSectionStartSymAndBlock( + Section &GraphSec, uint64_t Address, const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive) { + Block &B = + Data ? G->createContentBlock(GraphSec, StringRef(Data, Size), Address, + Alignment, 0) + : G->createZeroFillBlock(GraphSec, Size, Address, Alignment, 0); + auto &Sym = G->addAnonymousSymbol(B, 0, Size, false, IsLive); + assert(!AddrToCanonicalSymbol.count(Sym.getAddress()) && + "Anonymous block start symbol clashes with existing symbol address"); + AddrToCanonicalSymbol[Sym.getAddress()] = &Sym; +} + +Error MachOLinkGraphBuilder::graphifyRegularSymbols() { + + LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); + + /// We only have 256 section indexes: Use a vector rather than a map. + std::vector<std::vector<NormalizedSymbol *>> SecIndexToSymbols; + SecIndexToSymbols.resize(256); + + // Create commons, externs, and absolutes, and partition all other symbols by + // section. + for (auto &KV : IndexToSymbol) { + auto &NSym = *KV.second; + + switch (NSym.Type & MachO::N_TYPE) { + case MachO::N_UNDF: + if (NSym.Value) { + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous common symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addCommonSymbol( + *NSym.Name, NSym.S, getCommonSection(), NSym.Value, 0, + 1ull << MachO::GET_COMM_ALIGN(NSym.Desc), + NSym.Desc & MachO::N_NO_DEAD_STRIP); + } else { + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous external symbol at " + "index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addExternalSymbol(*NSym.Name, 0); + } + break; + case MachO::N_ABS: + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous absolute symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addAbsoluteSymbol( + *NSym.Name, NSym.Value, 0, Linkage::Strong, Scope::Default, + NSym.Desc & MachO::N_NO_DEAD_STRIP); + break; + case MachO::N_SECT: + SecIndexToSymbols[NSym.Sect - 1].push_back(&NSym); + break; + case MachO::N_PBUD: + return make_error<JITLinkError>( + "Unupported N_PBUD symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + case MachO::N_INDR: + return make_error<JITLinkError>( + "Unupported N_INDR symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + default: + return make_error<JITLinkError>( + "Unrecognized symbol type " + Twine(NSym.Type & MachO::N_TYPE) + + " for symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + } + } + + // Loop over sections performing regular graphification for those that + // don't have custom parsers. + for (auto &KV : IndexToSection) { + auto SecIndex = KV.first; + auto &NSec = KV.second; + + // Skip sections with custom parsers. + if (CustomSectionParserFunctions.count(NSec.GraphSection->getName())) { + LLVM_DEBUG({ + dbgs() << " Skipping section " << NSec.GraphSection->getName() + << " as it has a custom parser.\n"; + }); + continue; + } else + LLVM_DEBUG({ + dbgs() << " Processing section " << NSec.GraphSection->getName() + << "...\n"; + }); + + bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP; + bool SectionIsText = NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS; + + auto &SecNSymStack = SecIndexToSymbols[SecIndex]; + + // If this section is non-empty but there are no symbols covering it then + // create one block and anonymous symbol to cover the entire section. + if (SecNSymStack.empty()) { + if (NSec.Size > 0) { + LLVM_DEBUG({ + dbgs() << " Section non-empty, but contains no symbols. " + "Creating anonymous block to cover " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + NSec.Size) << "\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + NSec.Size, NSec.Alignment, + SectionIsNoDeadStrip); + } else + LLVM_DEBUG({ + dbgs() << " Section empty and contains no symbols. Skipping.\n"; + }); + continue; + } + + // Sort the symbol stack in by address, alt-entry status, scope, and name. + // We sort in reverse order so that symbols will be visited in the right + // order when we pop off the stack below. + llvm::sort(SecNSymStack, [](const NormalizedSymbol *LHS, + const NormalizedSymbol *RHS) { + if (LHS->Value != RHS->Value) + return LHS->Value > RHS->Value; + if (isAltEntry(*LHS) != isAltEntry(*RHS)) + return isAltEntry(*RHS); + if (LHS->S != RHS->S) + return static_cast<uint8_t>(LHS->S) < static_cast<uint8_t>(RHS->S); + return LHS->Name < RHS->Name; + }); + + // The first symbol in a section can not be an alt-entry symbol. + if (!SecNSymStack.empty() && isAltEntry(*SecNSymStack.back())) + return make_error<JITLinkError>( + "First symbol in " + NSec.GraphSection->getName() + " is alt-entry"); + + // If the section is non-empty but there is no symbol covering the start + // address then add an anonymous one. + if (SecNSymStack.back()->Value != NSec.Address) { + auto AnonBlockSize = SecNSymStack.back()->Value - NSec.Address; + LLVM_DEBUG({ + dbgs() << " Section start not covered by symbol. " + << "Creating anonymous block to cover [ " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + AnonBlockSize) << " ]\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + AnonBlockSize, NSec.Alignment, + SectionIsNoDeadStrip); + } + + // Visit section symbols in order by popping off the reverse-sorted stack, + // building blocks for each alt-entry chain and creating symbols as we go. + while (!SecNSymStack.empty()) { + SmallVector<NormalizedSymbol *, 8> BlockSyms; + + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + while (!SecNSymStack.empty() && + (isAltEntry(*SecNSymStack.back()) || + SecNSymStack.back()->Value == BlockSyms.back()->Value)) { + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + } + + // BlockNSyms now contains the block symbols in reverse canonical order. + JITTargetAddress BlockStart = BlockSyms.front()->Value; + JITTargetAddress BlockEnd = SecNSymStack.empty() + ? NSec.Address + NSec.Size + : SecNSymStack.back()->Value; + JITTargetAddress BlockOffset = BlockStart - NSec.Address; + JITTargetAddress BlockSize = BlockEnd - BlockStart; + + LLVM_DEBUG({ + dbgs() << " Creating block for " << formatv("{0:x16}", BlockStart) + << " -- " << formatv("{0:x16}", BlockEnd) << ": " + << NSec.GraphSection->getName() << " + " + << formatv("{0:x16}", BlockOffset) << " with " + << BlockSyms.size() << " symbol(s)...\n"; + }); + + Block &B = + NSec.Data + ? G->createContentBlock( + *NSec.GraphSection, + StringRef(NSec.Data + BlockOffset, BlockSize), BlockStart, + NSec.Alignment, BlockStart % NSec.Alignment) + : G->createZeroFillBlock(*NSec.GraphSection, BlockSize, + BlockStart, NSec.Alignment, + BlockStart % NSec.Alignment); + + Optional<JITTargetAddress> LastCanonicalAddr; + JITTargetAddress SymEnd = BlockEnd; + while (!BlockSyms.empty()) { + auto &NSym = *BlockSyms.back(); + BlockSyms.pop_back(); + + bool SymLive = + (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; + + LLVM_DEBUG({ + dbgs() << " " << formatv("{0:x16}", NSym.Value) << " -- " + << formatv("{0:x16}", SymEnd) << ": "; + if (!NSym.Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << NSym.Name; + if (SymLive) + dbgs() << " [no-dead-strip]"; + if (LastCanonicalAddr == NSym.Value) + dbgs() << " [non-canonical]"; + dbgs() << "\n"; + }); + + auto &Sym = + NSym.Name + ? G->addDefinedSymbol(B, NSym.Value - BlockStart, *NSym.Name, + SymEnd - NSym.Value, NSym.L, NSym.S, + SectionIsText, SymLive) + : G->addAnonymousSymbol(B, NSym.Value - BlockStart, + SymEnd - NSym.Value, SectionIsText, + SymLive); + NSym.GraphSymbol = &Sym; + if (LastCanonicalAddr != Sym.getAddress()) { + if (LastCanonicalAddr) + SymEnd = *LastCanonicalAddr; + LastCanonicalAddr = Sym.getAddress(); + setCanonicalSymbol(Sym); + } + } + } + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() { + // Graphify special sections. + for (auto &KV : IndexToSection) { + auto &NSec = KV.second; + + auto HI = CustomSectionParserFunctions.find(NSec.GraphSection->getName()); + if (HI != CustomSectionParserFunctions.end()) { + auto &Parse = HI->second; + if (auto Err = Parse(NSec)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h new file mode 100644 index 000000000000..e1123cd11048 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -0,0 +1,269 @@ +//===----- MachOLinkGraphBuilder.h - MachO LinkGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "EHFrameSupportImpl.h" +#include "JITLinkGeneric.h" +#include "llvm/Object/MachO.h" + +#include <list> + +namespace llvm { +namespace jitlink { + +class MachOLinkGraphBuilder { +public: + virtual ~MachOLinkGraphBuilder(); + Expected<std::unique_ptr<LinkGraph>> buildGraph(); + +protected: + class MachOEHFrameBinaryParser : public EHFrameBinaryParser { + public: + MachOEHFrameBinaryParser(MachOLinkGraphBuilder &Builder, + JITTargetAddress EHFrameAddress, + StringRef EHFrameContent, Section &EHFrameSection, + uint64_t CIEAlignment, uint64_t FDEAlignment, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : EHFrameBinaryParser(EHFrameAddress, EHFrameContent, + Builder.getGraph().getPointerSize(), + Builder.getGraph().getEndianness()), + Builder(Builder), EHFrameSection(EHFrameSection), + CIEAlignment(CIEAlignment), FDEAlignment(FDEAlignment), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + + Symbol *getSymbolAtAddress(JITTargetAddress Address) override { + if (auto *Sym = Builder.getSymbolByAddress(Address)) + if (Sym->getAddress() == Address) + return Sym; + return nullptr; + } + + Symbol &createCIERecord(JITTargetAddress RecordAddr, + StringRef RecordContent) override { + auto &G = Builder.getGraph(); + auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, + CIEAlignment, 0); + auto &CIESymbol = + G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); + Builder.setCanonicalSymbol(CIESymbol); + return CIESymbol; + } + + Expected<Symbol &> createFDERecord(JITTargetAddress RecordAddr, + StringRef RecordContent, Symbol &CIE, + size_t CIEOffset, Symbol &Func, + size_t FuncOffset, Symbol *LSDA, + size_t LSDAOffset) override { + auto &G = Builder.getGraph(); + auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, + FDEAlignment, 0); + + // Add edges to CIE, Func, and (conditionally) LSDA. + B.addEdge(FDEToCIERelocKind, CIEOffset, CIE, 0); + B.addEdge(FDEToTargetRelocKind, FuncOffset, Func, 0); + + if (LSDA) + B.addEdge(FDEToTargetRelocKind, LSDAOffset, *LSDA, 0); + + auto &FDESymbol = + G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); + + // Add a keep-alive relocation from the function to the FDE to ensure it + // is not dead stripped. + Func.getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + + return FDESymbol; + } + + private: + MachOLinkGraphBuilder &Builder; + Section &EHFrameSection; + uint64_t CIEAlignment; + uint64_t FDEAlignment; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; + }; + + struct NormalizedSymbol { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSymbol(Optional<StringRef> Name, uint64_t Value, uint8_t Type, + uint8_t Sect, uint16_t Desc, Linkage L, Scope S) + : Name(Name), Value(Value), Type(Type), Sect(Sect), Desc(Desc), L(L), + S(S) { + assert((!Name || !Name->empty()) && "Name must be none or non-empty"); + } + + public: + NormalizedSymbol(const NormalizedSymbol &) = delete; + NormalizedSymbol &operator=(const NormalizedSymbol &) = delete; + NormalizedSymbol(NormalizedSymbol &&) = delete; + NormalizedSymbol &operator=(NormalizedSymbol &&) = delete; + + Optional<StringRef> Name; + uint64_t Value = 0; + uint8_t Type = 0; + uint8_t Sect = 0; + uint16_t Desc = 0; + Linkage L = Linkage::Strong; + Scope S = Scope::Default; + Symbol *GraphSymbol = nullptr; + }; + + class NormalizedSection { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSection() = default; + + public: + Section *GraphSection = nullptr; + uint64_t Address = 0; + uint64_t Size = 0; + uint64_t Alignment = 0; + uint32_t Flags = 0; + const char *Data = nullptr; + }; + + using SectionParserFunction = std::function<Error(NormalizedSection &S)>; + + MachOLinkGraphBuilder(const object::MachOObjectFile &Obj); + + LinkGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomSectionParser(StringRef SectionName, + SectionParserFunction Parse); + + virtual Error addRelocations() = 0; + + /// Create a symbol. + template <typename... ArgTs> + NormalizedSymbol &createNormalizedSymbol(ArgTs &&... Args) { + NormalizedSymbol *Sym = reinterpret_cast<NormalizedSymbol *>( + Allocator.Allocate<NormalizedSymbol>()); + new (Sym) NormalizedSymbol(std::forward<ArgTs>(Args)...); + return *Sym; + } + + /// Index is zero-based (MachO section indexes are usually one-based) and + /// assumed to be in-range. Client is responsible for checking. + NormalizedSection &getSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + assert(I != IndexToSection.end() && "No section recorded at index"); + return I->second; + } + + /// Try to get the section at the given index. Will return an error if the + /// given index is out of range, or if no section has been added for the given + /// index. + Expected<NormalizedSection &> findSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + if (I == IndexToSection.end()) + return make_error<JITLinkError>("No section recorded for index " + + formatv("{0:u}", Index)); + return I->second; + } + + /// Try to get the symbol at the given index. Will return an error if the + /// given index is out of range, or if no symbol has been added for the given + /// index. + Expected<NormalizedSymbol &> findSymbolByIndex(uint64_t Index) { + if (Index >= IndexToSymbol.size()) + return make_error<JITLinkError>("Symbol index out of range"); + auto *Sym = IndexToSymbol[Index]; + if (!Sym) + return make_error<JITLinkError>("No symbol at index " + + formatv("{0:u}", Index)); + return *Sym; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or null if no such symbol exists. + Symbol *getSymbolByAddress(JITTargetAddress Address) { + auto I = AddrToCanonicalSymbol.upper_bound(Address); + if (I == AddrToCanonicalSymbol.begin()) + return nullptr; + return std::prev(I)->second; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or an error if no such symbol exists. + Expected<Symbol &> findSymbolByAddress(JITTargetAddress Address) { + auto *Sym = getSymbolByAddress(Address); + if (Sym) + if (Address < Sym->getAddress() + Sym->getSize()) + return *Sym; + return make_error<JITLinkError>("No symbol covering address " + + formatv("{0:x16}", Address)); + } + + static Linkage getLinkage(uint16_t Desc); + static Scope getScope(StringRef Name, uint8_t Type); + static bool isAltEntry(const NormalizedSymbol &NSym); + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + void setCanonicalSymbol(Symbol &Sym) { + auto *&CanonicalSymEntry = AddrToCanonicalSymbol[Sym.getAddress()]; + // There should be no symbol at this address, or, if there is, + // it should be a zero-sized symbol from an empty section (which + // we can safely override). + assert((!CanonicalSymEntry || CanonicalSymEntry->getSize() == 0) && + "Duplicate canonical symbol at address"); + CanonicalSymEntry = &Sym; + } + + Section &getCommonSection(); + void addSectionStartSymAndBlock(Section &GraphSec, uint64_t Address, + const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive); + + Error createNormalizedSections(); + Error createNormalizedSymbols(); + + /// Create graph blocks and symbols for externals, absolutes, commons and + /// all defined symbols in sections without custom parsers. + Error graphifyRegularSymbols(); + + /// Create graph blocks and symbols for all sections. + Error graphifySectionsWithCustomParsers(); + + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; + + const object::MachOObjectFile &Obj; + std::unique_ptr<LinkGraph> G; + + DenseMap<unsigned, NormalizedSection> IndexToSection; + Section *CommonSection = nullptr; + + DenseMap<uint32_t, NormalizedSymbol *> IndexToSymbol; + std::map<JITTargetAddress, Symbol *> AddrToCanonicalSymbol; + StringMap<SectionParserFunction> CustomSectionParserFunctions; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H diff --git a/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/lib/ExecutionEngine/JITLink/MachO_arm64.cpp new file mode 100644 index 000000000000..945343bff89d --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -0,0 +1,736 @@ +//===---- MachO_arm64.cpp - JIT linker implementation for MachO/arm64 -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO/arm64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" + +#include "BasicGOTAndStubsBuilder.h" +#include "MachOLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_arm64_Edges; + +namespace { + +class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { +public: + MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) + : MachOLinkGraphBuilder(Obj), + NumSymbols(Obj.getSymtabLoadCommand().nsyms) { + addCustomSectionParser( + "__eh_frame", [this](NormalizedSection &EHFrameSection) { + if (!EHFrameSection.Data) + return make_error<JITLinkError>( + "__eh_frame section is marked zero-fill"); + return MachOEHFrameBinaryParser( + *this, EHFrameSection.Address, + StringRef(EHFrameSection.Data, EHFrameSection.Size), + *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) + .addToGraph(); + }); + } + +private: + static Expected<MachOARM64RelocationKind> + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::ARM64_RELOC_UNSIGNED: + if (!RI.r_pcrel) { + if (RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + else if (RI.r_length == 2) + return Pointer32; + } + break; + case MachO::ARM64_RELOC_SUBTRACTOR: + // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta<W>'. + // They may be turned into NegDelta<W> by parsePairRelocation. + if (!RI.r_pcrel && RI.r_extern) { + if (RI.r_length == 2) + return Delta32; + else if (RI.r_length == 3) + return Delta64; + } + break; + case MachO::ARM64_RELOC_BRANCH26: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch26; + break; + case MachO::ARM64_RELOC_PAGE21: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Page21; + break; + case MachO::ARM64_RELOC_PAGEOFF12: + if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PageOffset12; + break; + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return GOTPage21; + break; + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: + if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return GOTPageOffset12; + break; + case MachO::ARM64_RELOC_POINTER_TO_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PointerToGOT; + break; + case MachO::ARM64_RELOC_ADDEND: + if (!RI.r_pcrel && !RI.r_extern && RI.r_length == 2) + return PairedAddend; + break; + } + + return make_error<JITLinkError>( + "Unsupported arm64 relocation: address=" + + formatv("{0:x8}", RI.r_address) + + ", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) + + ", kind=" + formatv("{0:x1}", RI.r_type) + + ", pc_rel=" + (RI.r_pcrel ? "true" : "false") + + ", extern=" + (RI.r_extern ? "true" : "false") + + ", length=" + formatv("{0:d}", RI.r_length)); + } + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); + return RI; + } + + using PairRelocInfo = + std::tuple<MachOARM64RelocationKind, Symbol *, uint64_t>; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected<PairRelocInfo> + parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error<JITLinkError>("arm64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_address != UnsignedRI.r_address) + return make_error<JITLinkError>("arm64 SUBTRACTOR and paired UNSIGNED " + "point to different addresses"); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error<JITLinkError>("length of arm64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + Symbol *FromSymbol; + if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum)) + FromSymbol = FromSymbolOrErr->GraphSymbol; + else + return FromSymbolOrErr.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const little64_t *)FixupContent; + else + FixupValue = *(const little32_t *)FixupContent; + + // Find 'ToSymbol' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + Symbol *ToSymbol = nullptr; + if (UnsignedRI.r_extern) { + // Find target symbol by symbol index. + if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum)) + ToSymbol = ToSymbolOrErr->GraphSymbol; + else + return ToSymbolOrErr.takeError(); + } else { + if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue)) + ToSymbol = &*ToSymbolOrErr; + else + return ToSymbolOrErr.takeError(); + FixupValue -= ToSymbol->getAddress(); + } + + MachOARM64RelocationKind DeltaKind; + Symbol *TargetSymbol; + uint64_t Addend; + if (&BlockToFix == &FromSymbol->getAddressable()) { + TargetSymbol = ToSymbol; + DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); + // FIXME: handle extern 'from'. + } else if (&BlockToFix == &ToSymbol->getAddressable()) { + TargetSymbol = &*FromSymbol; + DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); + } else { + // BlockToFix was neither FromSymbol nor ToSymbol. + return make_error<JITLinkError>("SUBTRACTOR relocation must fix up " + "either 'A' or 'B' (or a symbol in one " + "of their alt-entry groups)"); + } + + return PairRelocInfo(DeltaKind, TargetSymbol, Addend); + } + + Error addRelocations() override { + using namespace support; + auto &Obj = getObject(); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG({ + dbgs() << "Processing " << getMachOARM64RelocationKindName(*Kind) + << " relocation at " << format("0x%016" PRIx64, FixupAddress) + << "\n"; + }); + + // Find the block that the fixup points to. + Block *BlockToFix = nullptr; + { + auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress); + if (!SymbolToFixOrErr) + return SymbolToFixOrErr.takeError(); + BlockToFix = &SymbolToFixOrErr->getBlock(); + } + + if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > + BlockToFix->getAddress() + BlockToFix->getContent().size()) + return make_error<JITLinkError>( + "Relocation content extends past end of fixup block"); + + // Get a pointer to the fixup content. + const char *FixupContent = BlockToFix->getContent().data() + + (FixupAddress - BlockToFix->getAddress()); + + // The target symbol and addend will be populated by the switch below. + Symbol *TargetSymbol = nullptr; + uint64_t Addend = 0; + + if (*Kind == PairedAddend) { + // If this is an Addend relocation then process it and move to the + // paired reloc. + + Addend = RI.r_symbolnum; + + if (RelItr == RelEnd) + return make_error<JITLinkError>("Unpaired Addend reloc at " + + formatv("{0:x16}", FixupAddress)); + ++RelItr; + RI = getRelocationInfo(RelItr); + + Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + if (*Kind != Branch26 && *Kind != Page21 && *Kind != PageOffset12) + return make_error<JITLinkError>( + "Invalid relocation pair: Addend + " + + getMachOARM64RelocationKindName(*Kind)); + else + LLVM_DEBUG({ + dbgs() << " pair is " << getMachOARM64RelocationKindName(*Kind) + << "`\n"; + }); + + // Find the address of the value to fix up. + JITTargetAddress PairedFixupAddress = + SectionAddress + (uint32_t)RI.r_address; + if (PairedFixupAddress != FixupAddress) + return make_error<JITLinkError>("Paired relocation points at " + "different target"); + } + + switch (*Kind) { + case Branch26: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if ((Instr & 0x7fffffff) != 0x14000000) + return make_error<JITLinkError>("BRANCH26 target is not a B or BL " + "instruction with a zero addend"); + break; + } + case Pointer32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; + else + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); + break; + } + case Page21: + case GOTPage21: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if ((Instr & 0xffffffe0) != 0x90000000) + return make_error<JITLinkError>("PAGE21/GOTPAGE21 target is not an " + "ADRP instruction with a zero " + "addend"); + break; + } + case PageOffset12: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + break; + } + case GOTPageOffset12: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if ((Instr & 0xfffffc00) != 0xf9400000) + return make_error<JITLinkError>("GOTPAGEOFF12 target is not an LDR " + "immediate instruction with a zero " + "addend"); + break; + } + case PointerToGOT: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + break; + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + assert(TargetSymbol && "No target symbol from parsePairRelocation?"); + break; + } + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG({ + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Addend); + printEdge(dbgs(), *BlockToFix, GE, + getMachOARM64RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); + } + } + return Error::success(); + } + + unsigned NumSymbols = 0; +}; + +class MachO_arm64_GOTAndStubsBuilder + : public BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder> { +public: + MachO_arm64_GOTAndStubsBuilder(LinkGraph &G) + : BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder>(G) {} + + bool isGOTEdge(Edge &E) const { + return E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || + E.getKind() == PointerToGOT; + } + + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { + if (E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12) { + // Update the target, but leave the edge addend as-is. + E.setTarget(GOTEntry); + } else if (E.getKind() == PointerToGOT) { + E.setTarget(GOTEntry); + E.setKind(Delta32); + } else + llvm_unreachable("Not a GOT edge?"); + } + + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == Branch26 && !E.getTarget().isDefined(); + } + + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); + } + + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { + assert(E.getKind() == Branch26 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + E.setTarget(Stub); + } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); + return *GOTSection; + } + + Section &getStubsSection() { + if (!StubsSection) { + auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt); + } + return *StubsSection; + } + + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast<const char *>(StubContent), + sizeof(StubContent)); + } + + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[8]; + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachO_arm64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachO_arm64_GOTAndStubsBuilder::StubContent[8] = { + 0x10, 0x00, 0x00, 0x58, // LDR x16, <literal> + 0x00, 0x02, 0x1f, 0xd6 // BR x16 +}; + +} // namespace + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_arm64 : public JITLinker<MachOJITLinker_arm64> { + friend class JITLinker<MachOJITLinker_arm64>; + +public: + MachOJITLinker_arm64(std::unique_ptr<JITLinkContext> Ctx, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOARM64RelocationKindName(R); + } + + Expected<std::unique_ptr<LinkGraph>> + buildGraph(MemoryBufferRef ObjBuffer) override { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOLinkGraphBuilder_arm64(**MachOObj).buildGraph(); + } + + static Error targetOutOfRangeError(const Block &B, const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, B, E, getMachOARM64RelocationKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); + } + + static unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t LDRLiteralMask = 0x3ffffc00; + + // Check for a GPR LDR immediate with a zero embedded literal. + // If found, the top two bits contain the shift. + if ((Instr & LDRLiteralMask) == 0x39400000) + return Instr >> 30; + + // Check for a Neon LDR immediate of size 64-bit or less with a zero + // embedded literal. If found, the top two bits contain the shift. + if ((Instr & LDRLiteralMask) == 0x3d400000) + return Instr >> 30; + + // Check for a Neon LDR immediate of size 128-bit with a zero embedded + // literal. + constexpr uint32_t SizeBitsMask = 0xc0000000; + if ((Instr & (LDRLiteralMask | SizeBitsMask)) == 0x3dc00000) + return 4; + + return 0; + } + + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + using namespace support; + + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress & 0x3) == 0 && "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast<uint64_t>(Value) & 0x3) + return make_error<JITLinkError>("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return targetOutOfRangeError(B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + if (Value > std::numeric_limits<uint32_t>::max()) + return targetOutOfRangeError(B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: + case GOTPage21: { + assert(E.getAddend() == 0 && "PAGE21/GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + E.getTarget().getAddress() & ~static_cast<uint64_t>(4096 - 1); + uint64_t PCPage = B.getAddress() & ~static_cast<uint64_t>(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) + return targetOutOfRangeError(B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case PageOffset12: { + assert(E.getAddend() == 0 && "PAGEOFF12 with non-zero addend"); + uint64_t TargetOffset = E.getTarget().getAddress() & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error<JITLinkError>("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case GOTPageOffset12: { + assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend"); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + + uint32_t TargetOffset = E.getTarget().getAddress() & 0xfff; + assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); + uint32_t EncodedImm = (TargetOffset >> 3) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error<JITLinkError>("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return targetOutOfRangeError(B, E); + + uint32_t EncodedImm = (static_cast<uint32_t>(Delta) >> 2) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + uint64_t NullValue = 0; +}; + +void jitLink_MachO_arm64(std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + Triple TT("arm64-apple-ios"); + + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { + MachO_arm64_GOTAndStubsBuilder(G).run(); + return Error::success(); + }); + } + + if (auto Err = Ctx->modifyPassConfig(TT, Config)) + return Ctx->notifyFailed(std::move(Err)); + + // Construct a JITLinker and run the link function. + MachOJITLinker_arm64::link(std::move(Ctx), std::move(Config)); +} + +StringRef getMachOARM64RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch26: + return "Branch26"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case Page21: + return "Page21"; + case PageOffset12: + return "PageOffset12"; + case GOTPage21: + return "GOTPage21"; + case GOTPageOffset12: + return "GOTPageOffset12"; + case PointerToGOT: + return "PointerToGOT"; + case PairedAddend: + return "PairedAddend"; + case LDRLiteral19: + return "LDRLiteral19"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 4010678c6d33..d83787ffd598 100644 --- a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -13,7 +13,7 @@ #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" #include "BasicGOTAndStubsBuilder.h" -#include "MachOAtomGraphBuilder.h" +#include "MachOLinkGraphBuilder.h" #define DEBUG_TYPE "jitlink" @@ -23,16 +23,21 @@ using namespace llvm::jitlink::MachO_x86_64_Edges; namespace { -class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: - MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOAtomGraphBuilder(Obj), - NumSymbols(Obj.getSymtabLoadCommand().nsyms) { - addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { - return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), - EHFrameSection.getContent(), - EHFrameSection.getAddress(), NegDelta32, Delta64); - }); + MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOLinkGraphBuilder(Obj) { + addCustomSectionParser( + "__eh_frame", [this](NormalizedSection &EHFrameSection) { + if (!EHFrameSection.Data) + return make_error<JITLinkError>( + "__eh_frame section is marked zero-fill"); + return MachOEHFrameBinaryParser( + *this, EHFrameSection.Address, + StringRef(EHFrameSection.Data, EHFrameSection.Size), + *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) + .addToGraph(); + }); } private: @@ -40,8 +45,12 @@ private: getRelocationKind(const MachO::relocation_info &RI) { switch (RI.r_type) { case MachO::X86_64_RELOC_UNSIGNED: - if (!RI.r_pcrel && RI.r_length == 3) - return RI.r_extern ? Pointer64 : Pointer64Anon; + if (!RI.r_pcrel) { + if (RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + else if (RI.r_extern && RI.r_length == 2) + return Pointer32; + } break; case MachO::X86_64_RELOC_SIGNED: if (RI.r_pcrel && RI.r_length == 2) @@ -94,21 +103,10 @@ private: ", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) + ", kind=" + formatv("{0:x1}", RI.r_type) + ", pc_rel=" + (RI.r_pcrel ? "true" : "false") + - ", extern= " + (RI.r_extern ? "true" : "false") + + ", extern=" + (RI.r_extern ? "true" : "false") + ", length=" + formatv("{0:d}", RI.r_length)); } - Expected<Atom &> findAtomBySymbolIndex(const MachO::relocation_info &RI) { - auto &Obj = getObject(); - if (RI.r_symbolnum >= NumSymbols) - return make_error<JITLinkError>("Symbol index out of range"); - auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); - auto Name = SymI->getName(); - if (!Name) - return Name.takeError(); - return getGraph().getAtomByName(*Name); - } - MachO::relocation_info getRelocationInfo(const object::relocation_iterator RelItr) { MachO::any_relocation_info ARI = @@ -118,12 +116,12 @@ private: return RI; } - using PairRelocInfo = std::tuple<MachOX86RelocationKind, Atom *, uint64_t>; + using PairRelocInfo = std::tuple<MachOX86RelocationKind, Symbol *, uint64_t>; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, // returns the edge kind and addend to be used. Expected<PairRelocInfo> - parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, const MachO::relocation_info &SubRI, JITTargetAddress FixupAddress, const char *FixupContent, object::relocation_iterator &UnsignedRelItr, @@ -150,9 +148,11 @@ private: return make_error<JITLinkError>("length of x86_64 SUBTRACTOR and paired " "UNSIGNED reloc must match"); - auto FromAtom = findAtomBySymbolIndex(SubRI); - if (!FromAtom) - return FromAtom.takeError(); + Symbol *FromSymbol; + if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum)) + FromSymbol = FromSymbolOrErr->GraphSymbol; + else + return FromSymbolOrErr.takeError(); // Read the current fixup value. uint64_t FixupValue = 0; @@ -161,54 +161,60 @@ private: else FixupValue = *(const little32_t *)FixupContent; - // Find 'ToAtom' using symbol number or address, depending on whether the + // Find 'ToSymbol' using symbol number or address, depending on whether the // paired UNSIGNED relocation is extern. - Atom *ToAtom = nullptr; + Symbol *ToSymbol = nullptr; if (UnsignedRI.r_extern) { - // Find target atom by symbol index. - if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI)) - ToAtom = &*ToAtomOrErr; + // Find target symbol by symbol index. + if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum)) + ToSymbol = ToSymbolOrErr->GraphSymbol; else - return ToAtomOrErr.takeError(); + return ToSymbolOrErr.takeError(); } else { - if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue)) - ToAtom = &*ToAtomOrErr; + if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue)) + ToSymbol = &*ToSymbolOrErr; else - return ToAtomOrErr.takeError(); - FixupValue -= ToAtom->getAddress(); + return ToSymbolOrErr.takeError(); + FixupValue -= ToSymbol->getAddress(); } MachOX86RelocationKind DeltaKind; - Atom *TargetAtom; + Symbol *TargetSymbol; uint64_t Addend; - if (areLayoutLocked(AtomToFix, *FromAtom)) { - TargetAtom = ToAtom; + if (&BlockToFix == &FromSymbol->getAddressable()) { + TargetSymbol = ToSymbol; DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; - Addend = FixupValue + (FixupAddress - FromAtom->getAddress()); + Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); // FIXME: handle extern 'from'. - } else if (areLayoutLocked(AtomToFix, *ToAtom)) { - TargetAtom = &*FromAtom; + } else if (&BlockToFix == &ToSymbol->getAddressable()) { + TargetSymbol = FromSymbol; DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; - Addend = FixupValue - (FixupAddress - ToAtom->getAddress()); + Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); } else { - // AtomToFix was neither FromAtom nor ToAtom. + // BlockToFix was neither FromSymbol nor ToSymbol. return make_error<JITLinkError>("SUBTRACTOR relocation must fix up " - "either 'A' or 'B' (or an atom in one " - "of their alt-entry groups)"); + "either 'A' or 'B' (or a symbol in one " + "of their alt-entry chains)"); } - return PairRelocInfo(DeltaKind, TargetAtom, Addend); + return PairRelocInfo(DeltaKind, TargetSymbol, Addend); } Error addRelocations() override { using namespace support; - auto &G = getGraph(); auto &Obj = getObject(); for (auto &S : Obj.sections()) { JITTargetAddress SectionAddress = S.getAddress(); + if (S.isVirtual()) { + if (S.relocation_begin() != S.relocation_end()) + return make_error<JITLinkError>("Virtual section contains " + "relocations"); + continue; + } + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); RelItr != RelEnd; ++RelItr) { @@ -227,26 +233,26 @@ private: << format("0x%016" PRIx64, FixupAddress) << "\n"; }); - // Find the atom that the fixup points to. - DefinedAtom *AtomToFix = nullptr; + // Find the block that the fixup points to. + Block *BlockToFix = nullptr; { - auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); - if (!AtomToFixOrErr) - return AtomToFixOrErr.takeError(); - AtomToFix = &*AtomToFixOrErr; + auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress); + if (!SymbolToFixOrErr) + return SymbolToFixOrErr.takeError(); + BlockToFix = &SymbolToFixOrErr->getBlock(); } if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > - AtomToFix->getAddress() + AtomToFix->getContent().size()) + BlockToFix->getAddress() + BlockToFix->getContent().size()) return make_error<JITLinkError>( - "Relocation content extends past end of fixup atom"); + "Relocation extends past end of fixup block"); // Get a pointer to the fixup content. - const char *FixupContent = AtomToFix->getContent().data() + - (FixupAddress - AtomToFix->getAddress()); + const char *FixupContent = BlockToFix->getContent().data() + + (FixupAddress - BlockToFix->getAddress()); - // The target atom and addend will be populated by the switch below. - Atom *TargetAtom = nullptr; + // The target symbol and addend will be populated by the switch below. + Symbol *TargetSymbol = nullptr; uint64_t Addend = 0; switch (*Kind) { @@ -254,46 +260,53 @@ private: case PCRel32: case PCRel32GOTLoad: case PCRel32GOT: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; break; case Pointer64: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle64_t *)FixupContent; break; case Pointer64Anon: { JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case PCRel32Minus1: case PCRel32Minus2: case PCRel32Minus4: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent + (1 << (*Kind - PCRel32Minus1)); break; case PCRel32Anon: { JITTargetAddress TargetAddress = FixupAddress + 4 + *(const ulittle32_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case PCRel32Minus1Anon: @@ -303,11 +316,11 @@ private: static_cast<JITTargetAddress>(1ULL << (*Kind - PCRel32Minus1Anon)); JITTargetAddress TargetAddress = FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case Delta32: @@ -318,12 +331,12 @@ private: // NegDelta32/NegDelta64, depending on the direction of the // subtraction) along with the addend. auto PairInfo = - parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, FixupContent, ++RelItr, RelEnd); if (!PairInfo) return PairInfo.takeError(); - std::tie(*Kind, TargetAtom, Addend) = *PairInfo; - assert(TargetAtom && "No target atom from parsePairRelocation?"); + std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } default: @@ -332,41 +345,38 @@ private: } LLVM_DEBUG({ - Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom, + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - printEdge(dbgs(), *AtomToFix, GE, + printEdge(dbgs(), *BlockToFix, GE, getMachOX86RelocationKindName(*Kind)); dbgs() << "\n"; }); - AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), - *TargetAtom, Addend); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); } } return Error::success(); } - - unsigned NumSymbols = 0; }; class MachO_x86_64_GOTAndStubsBuilder : public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> { public: - MachO_x86_64_GOTAndStubsBuilder(AtomGraph &G) + MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {} bool isGOTEdge(Edge &E) const { return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; } - DefinedAtom &createGOTEntry(Atom &Target) { - auto &GOTEntryAtom = G.addAnonymousAtom(getGOTSection(), 0x0, 8); - GOTEntryAtom.setContent( - StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), 8)); - GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); - return GOTEntryAtom; + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); } - void fixGOTEdge(Edge &E, Atom &GOTEntry) { + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && "Not a GOT edge?"); E.setKind(PCRel32); @@ -378,19 +388,16 @@ public: return E.getKind() == Branch32 && !E.getTarget().isDefined(); } - DefinedAtom &createStub(Atom &Target) { - auto &StubAtom = G.addAnonymousAtom(getStubsSection(), 0x0, 2); - StubAtom.setContent( - StringRef(reinterpret_cast<const char *>(StubContent), 6)); - + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. - auto &GOTEntryAtom = getGOTEntryAtom(Target); - StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); - - return StubAtom; + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); } - void fixExternalBranchEdge(Edge &E, Atom &Stub) { + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); E.setTarget(Stub); @@ -399,7 +406,7 @@ public: private: Section &getGOTSection() { if (!GOTSection) - GOTSection = &G.createSection("$__GOT", 8, sys::Memory::MF_READ, false); + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); return *GOTSection; } @@ -407,11 +414,21 @@ private: if (!StubsSection) { auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( sys::Memory::MF_READ | sys::Memory::MF_EXEC); - StubsSection = &G.createSection("$__STUBS", 8, StubsProt, false); + StubsSection = &G.createSection("$__STUBS", StubsProt); } return *StubsSection; } + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast<const char *>(StubContent), + sizeof(StubContent)); + } + static const uint8_t NullGOTEntryContent[8]; static const uint8_t StubContent[6]; Section *GOTSection = nullptr; @@ -440,30 +457,31 @@ private: return getMachOX86RelocationKindName(R); } - Expected<std::unique_ptr<AtomGraph>> + Expected<std::unique_ptr<LinkGraph>> buildGraph(MemoryBufferRef ObjBuffer) override { auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); if (!MachOObj) return MachOObj.takeError(); - return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph(); } - static Error targetOutOfRangeError(const Atom &A, const Edge &E) { + static Error targetOutOfRangeError(const Block &B, const Edge &E) { std::string ErrMsg; { raw_string_ostream ErrStream(ErrMsg); ErrStream << "Relocation target out of range: "; - printEdge(ErrStream, A, E, getMachOX86RelocationKindName(E.getKind())); + printEdge(ErrStream, B, E, getMachOX86RelocationKindName(E.getKind())); ErrStream << "\n"; } return make_error<JITLinkError>(std::move(ErrMsg)); } - Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + using namespace support; - char *FixupPtr = AtomWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { case Branch32: @@ -473,7 +491,7 @@ private: E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -491,7 +509,7 @@ private: E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -503,7 +521,7 @@ private: E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -520,12 +538,19 @@ private: if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; } else *(little64_t *)FixupPtr = Value; break; } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + if (Value > std::numeric_limits<uint32_t>::max()) + return targetOutOfRangeError(B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } default: llvm_unreachable("Unrecognized edge kind"); } @@ -545,10 +570,10 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else - Config.PrePrunePasses.push_back(markAllAtomsLive); + Config.PrePrunePasses.push_back(markAllSymbolsLive); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error { + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { MachO_x86_64_GOTAndStubsBuilder(G).run(); return Error::success(); }); @@ -565,6 +590,8 @@ StringRef getMachOX86RelocationKindName(Edge::Kind R) { switch (R) { case Branch32: return "Branch32"; + case Pointer32: + return "Pointer32"; case Pointer64: return "Pointer64"; case Pointer64Anon: diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp index 08815b7a80ae..94741f5f01d5 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -23,7 +23,7 @@ #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/MutexGuard.h" +#include <mutex> using namespace llvm; @@ -88,7 +88,7 @@ MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> TM, } MCJIT::~MCJIT() { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); Dyld.deregisterEHFrames(); @@ -100,7 +100,7 @@ MCJIT::~MCJIT() { } void MCJIT::addModule(std::unique_ptr<Module> M) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); if (M->getDataLayout().isDefault()) M->setDataLayout(getDataLayout()); @@ -109,7 +109,7 @@ void MCJIT::addModule(std::unique_ptr<Module> M) { } bool MCJIT::removeModule(Module *M) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); return OwnedModules.removeModule(M); } @@ -136,14 +136,14 @@ void MCJIT::addArchive(object::OwningBinary<object::Archive> A) { } void MCJIT::setObjectCache(ObjectCache* NewCache) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); ObjCache = NewCache; } std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) { assert(M && "Can not emit a null module"); - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // Materialize all globals in the module if they have not been // materialized already. @@ -185,7 +185,7 @@ std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) { void MCJIT::generateCodeForModule(Module *M) { // Get a thread lock to make sure we aren't trying to load multiple times - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // This must be a module which has already been added to this MCJIT instance. assert(OwnedModules.ownsModule(M) && @@ -234,7 +234,7 @@ void MCJIT::generateCodeForModule(Module *M) { } void MCJIT::finalizeLoadedModules() { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // Resolve any outstanding relocations. Dyld.resolveRelocations(); @@ -250,7 +250,7 @@ void MCJIT::finalizeLoadedModules() { // FIXME: Rename this. void MCJIT::finalizeObject() { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // Generate code for module is going to move objects out of the 'added' list, // so we need to copy that out before using it: @@ -265,7 +265,7 @@ void MCJIT::finalizeObject() { } void MCJIT::finalizeModule(Module *M) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // This must be a module which has already been added to this MCJIT instance. assert(OwnedModules.ownsModule(M) && "MCJIT::finalizeModule: Unknown module."); @@ -292,7 +292,7 @@ Module *MCJIT::findModuleForSymbol(const std::string &Name, if (DemangledName[0] == getDataLayout().getGlobalPrefix()) DemangledName = DemangledName.substr(1); - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // If it hasn't already been generated, see if it's in one of our modules. for (ModulePtrSet::iterator I = OwnedModules.begin_added(), @@ -332,7 +332,7 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, JITSymbol MCJIT::findSymbol(const std::string &Name, bool CheckFunctionsOnly) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // First, check to see if we already have this symbol. if (auto Sym = findExistingSymbol(Name)) @@ -388,7 +388,7 @@ JITSymbol MCJIT::findSymbol(const std::string &Name, } uint64_t MCJIT::getGlobalValueAddress(const std::string &Name) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); uint64_t Result = getSymbolAddress(Name, false); if (Result != 0) finalizeLoadedModules(); @@ -396,7 +396,7 @@ uint64_t MCJIT::getGlobalValueAddress(const std::string &Name) { } uint64_t MCJIT::getFunctionAddress(const std::string &Name) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); uint64_t Result = getSymbolAddress(Name, true); if (Result != 0) finalizeLoadedModules(); @@ -405,7 +405,7 @@ uint64_t MCJIT::getFunctionAddress(const std::string &Name) { // Deprecated. Use getFunctionAddress instead. void *MCJIT::getPointerToFunction(Function *F) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); Mangler Mang; SmallString<128> Name; @@ -632,14 +632,14 @@ void *MCJIT::getPointerToNamedFunction(StringRef Name, bool AbortOnFailure) { void MCJIT::RegisterJITEventListener(JITEventListener *L) { if (!L) return; - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); EventListeners.push_back(L); } void MCJIT::UnregisterJITEventListener(JITEventListener *L) { if (!L) return; - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); auto I = find(reverse(EventListeners), L); if (I != EventListeners.rend()) { std::swap(*I, EventListeners.back()); @@ -651,7 +651,7 @@ void MCJIT::notifyObjectLoaded(const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &L) { uint64_t Key = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Obj.getData().data())); - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); MemMgr->notifyObjectLoaded(this, Obj); for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { EventListeners[I]->notifyObjectLoaded(Key, Obj, L); @@ -661,7 +661,7 @@ void MCJIT::notifyObjectLoaded(const object::ObjectFile &Obj, void MCJIT::notifyFreeingObject(const object::ObjectFile &Obj) { uint64_t Key = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Obj.getData().data())); - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); for (JITEventListener *L : EventListeners) L->notifyFreeingObject(Key); } diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp index 2ad9d24555f3..bb5d96051da9 100644 --- a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp +++ b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp @@ -177,7 +177,7 @@ void OProfileJITEventListener::notifyFreeingObject(ObjectKey Key) { namespace llvm { JITEventListener *JITEventListener::createOProfileJITEventListener() { - return new OProfileJITEventListener(llvm::make_unique<OProfileWrapper>()); + return new OProfileJITEventListener(std::make_unique<OProfileWrapper>()); } } // namespace llvm diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp index 1a2667736926..b78d2531382d 100644 --- a/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp +++ b/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp @@ -17,11 +17,11 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/Mutex.h" -#include "llvm/Support/MutexGuard.h" #include "llvm/Support/raw_ostream.h" #include <cstring> #include <dirent.h> #include <fcntl.h> +#include <mutex> #include <stddef.h> #include <sys/stat.h> #include <unistd.h> @@ -54,7 +54,7 @@ bool OProfileWrapper::initialize() { using namespace llvm; using namespace llvm::sys; - MutexGuard Guard(OProfileInitializationMutex); + std::lock_guard<sys::Mutex> Guard(OProfileInitializationMutex); if (Initialized) return OpenAgentFunc != 0; diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 99bf53bc3afa..75ddbc30445d 100644 --- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -54,11 +54,12 @@ static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM, llvm_unreachable("Unsupported global type"); }; - auto NewTSMod = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs); - auto &M = *NewTSMod.getModule(); - M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str()); + auto NewTSM = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs); + NewTSM.withModuleDo([&](Module &M) { + M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str()); + }); - return NewTSMod; + return NewTSM; } namespace llvm { @@ -117,39 +118,44 @@ void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) { this->Partition = std::move(Partition); } +void CompileOnDemandLayer::setImplMap(ImplSymbolMap *Imp) { + this->AliaseeImpls = Imp; +} void CompileOnDemandLayer::emit(MaterializationResponsibility R, ThreadSafeModule TSM) { - assert(TSM.getModule() && "Null module"); + assert(TSM && "Null module"); auto &ES = getExecutionSession(); - auto &M = *TSM.getModule(); - - // First, do some cleanup on the module: - cleanUpModule(M); - // Now sort the callables and non-callables, build re-exports and lodge the + // Sort the callables and non-callables, build re-exports and lodge the // actual module with the implementation dylib. auto &PDR = getPerDylibResources(R.getTargetJITDylib()); - MangleAndInterner Mangle(ES, M.getDataLayout()); SymbolAliasMap NonCallables; SymbolAliasMap Callables; - for (auto &GV : M.global_values()) { - if (GV.isDeclaration() || GV.hasLocalLinkage() || GV.hasAppendingLinkage()) - continue; - - auto Name = Mangle(GV.getName()); - auto Flags = JITSymbolFlags::fromGlobalValue(GV); - if (Flags.isCallable()) - Callables[Name] = SymbolAliasMapEntry(Name, Flags); - else - NonCallables[Name] = SymbolAliasMapEntry(Name, Flags); - } + TSM.withModuleDo([&](Module &M) { + // First, do some cleanup on the module: + cleanUpModule(M); + + MangleAndInterner Mangle(ES, M.getDataLayout()); + for (auto &GV : M.global_values()) { + if (GV.isDeclaration() || GV.hasLocalLinkage() || + GV.hasAppendingLinkage()) + continue; + + auto Name = Mangle(GV.getName()); + auto Flags = JITSymbolFlags::fromGlobalValue(GV); + if (Flags.isCallable()) + Callables[Name] = SymbolAliasMapEntry(Name, Flags); + else + NonCallables[Name] = SymbolAliasMapEntry(Name, Flags); + } + }); // Create a partitioning materialization unit and lodge it with the // implementation dylib. if (auto Err = PDR.getImplDylib().define( - llvm::make_unique<PartitioningIRMaterializationUnit>( + std::make_unique<PartitioningIRMaterializationUnit>( ES, std::move(TSM), R.getVModuleKey(), *this))) { ES.reportError(std::move(Err)); R.failMaterialization(); @@ -158,7 +164,7 @@ void CompileOnDemandLayer::emit(MaterializationResponsibility R, R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), true)); R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), - std::move(Callables))); + std::move(Callables), AliaseeImpls)); } CompileOnDemandLayer::PerDylibResources & @@ -239,14 +245,16 @@ void CompileOnDemandLayer::emitPartition( // memory manager instance to the linking layer. auto &ES = getExecutionSession(); - GlobalValueSet RequestedGVs; for (auto &Name : R.getRequestedSymbols()) { assert(Defs.count(Name) && "No definition for symbol"); RequestedGVs.insert(Defs[Name]); } - auto GVsToExtract = Partition(RequestedGVs); + /// Perform partitioning with the context lock held, since the partition + /// function is allowed to access the globals to compute the partition. + auto GVsToExtract = + TSM.withModuleDo([&](Module &M) { return Partition(RequestedGVs); }); // Take a 'None' partition to mean the whole module (as opposed to an empty // partition, which means "materialize nothing"). Emit the whole module @@ -259,43 +267,52 @@ void CompileOnDemandLayer::emitPartition( // If the partition is empty, return the whole module to the symbol table. if (GVsToExtract->empty()) { - R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>( + R.replace(std::make_unique<PartitioningIRMaterializationUnit>( std::move(TSM), R.getSymbols(), std::move(Defs), *this)); return; } // Ok -- we actually need to partition the symbols. Promote the symbol - // linkages/names. - // FIXME: We apply this once per partitioning. It's safe, but overkill. - { - auto PromotedGlobals = PromoteSymbols(*TSM.getModule()); - if (!PromotedGlobals.empty()) { - MangleAndInterner Mangle(ES, TSM.getModule()->getDataLayout()); - SymbolFlagsMap SymbolFlags; - for (auto &GV : PromotedGlobals) - SymbolFlags[Mangle(GV->getName())] = - JITSymbolFlags::fromGlobalValue(*GV); - if (auto Err = R.defineMaterializing(SymbolFlags)) { - ES.reportError(std::move(Err)); - R.failMaterialization(); - return; - } - } + // linkages/names, expand the partition to include any required symbols + // (i.e. symbols that can't be separated from our partition), and + // then extract the partition. + // + // FIXME: We apply this promotion once per partitioning. It's safe, but + // overkill. + + auto ExtractedTSM = + TSM.withModuleDo([&](Module &M) -> Expected<ThreadSafeModule> { + auto PromotedGlobals = PromoteSymbols(M); + if (!PromotedGlobals.empty()) { + MangleAndInterner Mangle(ES, M.getDataLayout()); + SymbolFlagsMap SymbolFlags; + for (auto &GV : PromotedGlobals) + SymbolFlags[Mangle(GV->getName())] = + JITSymbolFlags::fromGlobalValue(*GV); + if (auto Err = R.defineMaterializing(SymbolFlags)) + return std::move(Err); + } + + expandPartition(*GVsToExtract); + + // Extract the requested partiton (plus any necessary aliases) and + // put the rest back into the impl dylib. + auto ShouldExtract = [&](const GlobalValue &GV) -> bool { + return GVsToExtract->count(&GV); + }; + + return extractSubModule(TSM, ".submodule", ShouldExtract); + }); + + if (!ExtractedTSM) { + ES.reportError(ExtractedTSM.takeError()); + R.failMaterialization(); + return; } - expandPartition(*GVsToExtract); - - // Extract the requested partiton (plus any necessary aliases) and - // put the rest back into the impl dylib. - auto ShouldExtract = [&](const GlobalValue &GV) -> bool { - return GVsToExtract->count(&GV); - }; - - auto ExtractedTSM = extractSubModule(TSM, ".submodule", ShouldExtract); - R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>( + R.replace(std::make_unique<PartitioningIRMaterializationUnit>( ES, std::move(TSM), R.getVModuleKey(), *this)); - - BaseLayer.emit(std::move(R), std::move(ExtractedTSM)); + BaseLayer.emit(std::move(R), std::move(*ExtractedTSM)); } } // end namespace orc diff --git a/lib/ExecutionEngine/Orc/CompileUtils.cpp b/lib/ExecutionEngine/Orc/CompileUtils.cpp index d46b6fcf9a5f..f8251627a4ef 100644 --- a/lib/ExecutionEngine/Orc/CompileUtils.cpp +++ b/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -42,7 +42,7 @@ SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { PM.run(M); } - auto ObjBuffer = llvm::make_unique<SmallVectorMemoryBuffer>( + auto ObjBuffer = std::make_unique<SmallVectorMemoryBuffer>( std::move(ObjBufferSV), "<in memory object compiled from " + M.getModuleIdentifier() + ">"); diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index dac37e030e0c..5c7d888c2d6e 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -151,6 +151,8 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { } raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { + if (Flags.hasError()) + OS << "[*ERROR*]"; if (Flags.isCallable()) OS << "[Callable]"; else @@ -224,7 +226,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { for (auto &KV : Aliases) OS << " " << *KV.first << ": " << KV.second.Aliasee << " " << KV.second.AliasFlags; - OS << " }\n"; + OS << " }"; return OS; } @@ -238,15 +240,18 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { return OS << "Materializing"; case SymbolState::Resolved: return OS << "Resolved"; + case SymbolState::Emitted: + return OS << "Emitted"; case SymbolState::Ready: return OS << "Ready"; } llvm_unreachable("Invalid state"); } -FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols) +FailedToMaterialize::FailedToMaterialize( + std::shared_ptr<SymbolDependenceMap> Symbols) : Symbols(std::move(Symbols)) { - assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); + assert(!this->Symbols->empty() && "Can not fail to resolve an empty set"); } std::error_code FailedToMaterialize::convertToErrorCode() const { @@ -254,7 +259,7 @@ std::error_code FailedToMaterialize::convertToErrorCode() const { } void FailedToMaterialize::log(raw_ostream &OS) const { - OS << "Failed to materialize symbols: " << Symbols; + OS << "Failed to materialize symbols: " << *Symbols; } SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) @@ -367,35 +372,35 @@ SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { return JD.getRequestedSymbols(SymbolFlags); } -void MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { +Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { LLVM_DEBUG({ dbgs() << "In " << JD.getName() << " resolving " << Symbols << "\n"; }); #ifndef NDEBUG for (auto &KV : Symbols) { + auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; auto I = SymbolFlags.find(KV.first); assert(I != SymbolFlags.end() && "Resolving symbol outside this responsibility set"); - if (I->second.isWeak()) - assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && - "Resolving symbol with incorrect flags"); - else - assert(I->second == KV.second.getFlags() && - "Resolving symbol with incorrect flags"); + assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && + "Resolving symbol with incorrect flags"); } #endif - JD.resolve(Symbols); + return JD.resolve(Symbols); } -void MaterializationResponsibility::notifyEmitted() { +Error MaterializationResponsibility::notifyEmitted() { LLVM_DEBUG({ dbgs() << "In " << JD.getName() << " emitting " << SymbolFlags << "\n"; }); - JD.emit(SymbolFlags); + if (auto Err = JD.emit(SymbolFlags)) + return Err; + SymbolFlags.clear(); + return Error::success(); } Error MaterializationResponsibility::defineMaterializing( @@ -417,12 +422,13 @@ void MaterializationResponsibility::failMaterialization() { << SymbolFlags << "\n"; }); - SymbolNameSet FailedSymbols; - for (auto &KV : SymbolFlags) - FailedSymbols.insert(KV.first); + JITDylib::FailedSymbolsWorklist Worklist; - JD.notifyFailed(FailedSymbols); + for (auto &KV : SymbolFlags) + Worklist.push_back(std::make_pair(&JD, KV.first)); SymbolFlags.clear(); + + JD.notifyFailed(std::move(Worklist)); } void MaterializationResponsibility::replace( @@ -485,8 +491,9 @@ StringRef AbsoluteSymbolsMaterializationUnit::getName() const { void AbsoluteSymbolsMaterializationUnit::materialize( MaterializationResponsibility R) { - R.notifyResolved(Symbols); - R.notifyEmitted(); + // No dependencies, so these calls can't fail. + cantFail(R.notifyResolved(Symbols)); + cantFail(R.notifyEmitted()); } void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, @@ -625,6 +632,7 @@ void ReExportsMaterializationUnit::materialize( }; auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) { + auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); if (Result) { SymbolMap ResolutionMap; for (auto &KV : QueryInfo->Aliases) { @@ -633,10 +641,17 @@ void ReExportsMaterializationUnit::materialize( ResolutionMap[KV.first] = JITEvaluatedSymbol( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } - QueryInfo->R.notifyResolved(ResolutionMap); - QueryInfo->R.notifyEmitted(); + if (auto Err = QueryInfo->R.notifyResolved(ResolutionMap)) { + ES.reportError(std::move(Err)); + QueryInfo->R.failMaterialization(); + return; + } + if (auto Err = QueryInfo->R.notifyEmitted()) { + ES.reportError(std::move(Err)); + QueryInfo->R.failMaterialization(); + return; + } } else { - auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); ES.reportError(Result.takeError()); QueryInfo->R.failMaterialization(); } @@ -694,7 +709,7 @@ ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, Allow(std::move(Allow)) {} Expected<SymbolNameSet> -ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) { +ReexportsGenerator::tryToGenerate(JITDylib &JD, const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolAliasMap AliasMap; @@ -716,6 +731,19 @@ ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) { return Added; } +JITDylib::DefinitionGenerator::~DefinitionGenerator() {} + +void JITDylib::removeGenerator(DefinitionGenerator &G) { + ES.runSessionLocked([&]() { + auto I = std::find_if(DefGenerators.begin(), DefGenerators.end(), + [&](const std::unique_ptr<DefinitionGenerator> &H) { + return H.get() == &G; + }); + assert(I != DefGenerators.end() && "Generator not found"); + DefGenerators.erase(I); + }); +} + Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { return ES.runSessionLocked([&]() -> Error { std::vector<SymbolTable::iterator> AddedSyms; @@ -823,26 +851,52 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, assert(Symbols[Name].isInMaterializationPhase() && "Can not add dependencies for a symbol that is not materializing"); + // If Name is already in an error state then just bail out. + if (Symbols[Name].getFlags().hasError()) + return; + auto &MI = MaterializingInfos[Name]; - assert(!MI.IsEmitted && "Can not add dependencies to an emitted symbol"); + assert(Symbols[Name].getState() != SymbolState::Emitted && + "Can not add dependencies to an emitted symbol"); + bool DependsOnSymbolInErrorState = false; + + // Register dependencies, record whether any depenendency is in the error + // state. for (auto &KV : Dependencies) { assert(KV.first && "Null JITDylib in dependency?"); auto &OtherJITDylib = *KV.first; auto &DepsOnOtherJITDylib = MI.UnemittedDependencies[&OtherJITDylib]; for (auto &OtherSymbol : KV.second) { + + // Check the sym entry for the dependency. + auto OtherSymI = OtherJITDylib.Symbols.find(OtherSymbol); + #ifndef NDEBUG - // Assert that this symbol exists and has not been emitted already. - auto SymI = OtherJITDylib.Symbols.find(OtherSymbol); - assert(SymI != OtherJITDylib.Symbols.end() && - (SymI->second.getState() != SymbolState::Ready && - "Dependency on emitted symbol")); + // Assert that this symbol exists and has not reached the ready state + // already. + assert(OtherSymI != OtherJITDylib.Symbols.end() && + (OtherSymI->second.getState() != SymbolState::Ready && + "Dependency on emitted/ready symbol")); #endif + auto &OtherSymEntry = OtherSymI->second; + + // If the dependency is in an error state then note this and continue, + // we will move this symbol to the error state below. + if (OtherSymEntry.getFlags().hasError()) { + DependsOnSymbolInErrorState = true; + continue; + } + + // If the dependency was not in the error state then add it to + // our list of dependencies. + assert(OtherJITDylib.MaterializingInfos.count(OtherSymbol) && + "No MaterializingInfo for dependency"); auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; - if (OtherMI.IsEmitted) + if (OtherSymEntry.getState() == SymbolState::Emitted) transferEmittedNodeDependencies(MI, Name, OtherMI); else if (&OtherJITDylib != this || OtherSymbol != Name) { OtherMI.Dependants[this].insert(Name); @@ -853,63 +907,142 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, if (DepsOnOtherJITDylib.empty()) MI.UnemittedDependencies.erase(&OtherJITDylib); } + + // If this symbol dependended on any symbols in the error state then move + // this symbol to the error state too. + if (DependsOnSymbolInErrorState) + Symbols[Name].setFlags(Symbols[Name].getFlags() | JITSymbolFlags::HasError); } -void JITDylib::resolve(const SymbolMap &Resolved) { - auto CompletedQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet CompletedQueries; +Error JITDylib::resolve(const SymbolMap &Resolved) { + SymbolNameSet SymbolsInErrorState; + AsynchronousSymbolQuerySet CompletedQueries; + + ES.runSessionLocked([&, this]() { + struct WorklistEntry { + SymbolTable::iterator SymI; + JITEvaluatedSymbol ResolvedSym; + }; + + std::vector<WorklistEntry> Worklist; + Worklist.reserve(Resolved.size()); + + // Build worklist and check for any symbols in the error state. for (const auto &KV : Resolved) { - auto &Name = KV.first; - auto Sym = KV.second; - auto I = Symbols.find(Name); + assert(!KV.second.getFlags().hasError() && + "Resolution result can not have error flag set"); - assert(I != Symbols.end() && "Symbol not found"); - assert(!I->second.hasMaterializerAttached() && + auto SymI = Symbols.find(KV.first); + + assert(SymI != Symbols.end() && "Symbol not found"); + assert(!SymI->second.hasMaterializerAttached() && "Resolving symbol with materializer attached?"); - assert(I->second.getState() == SymbolState::Materializing && + assert(SymI->second.getState() == SymbolState::Materializing && "Symbol should be materializing"); - assert(I->second.getAddress() == 0 && "Symbol has already been resolved"); + assert(SymI->second.getAddress() == 0 && + "Symbol has already been resolved"); + + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(KV.first); + else { + auto Flags = KV.second.getFlags(); + Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); + assert(Flags == (SymI->second.getFlags() & + ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && + "Resolved flags should match the declared flags"); + + Worklist.push_back( + {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); + } + } + + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) + return; + + while (!Worklist.empty()) { + auto SymI = Worklist.back().SymI; + auto ResolvedSym = Worklist.back().ResolvedSym; + Worklist.pop_back(); - assert((Sym.getFlags() & ~JITSymbolFlags::Weak) == - (I->second.getFlags() & ~JITSymbolFlags::Weak) && - "Resolved flags should match the declared flags"); + auto &Name = SymI->first; - // Once resolved, symbols can never be weak. - JITSymbolFlags ResolvedFlags = Sym.getFlags(); - ResolvedFlags &= ~JITSymbolFlags::Weak; - I->second.setAddress(Sym.getAddress()); - I->second.setFlags(ResolvedFlags); - I->second.setState(SymbolState::Resolved); + // Resolved symbols can not be weak: discard the weak flag. + JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags(); + SymI->second.setAddress(ResolvedSym.getAddress()); + SymI->second.setFlags(ResolvedFlags); + SymI->second.setState(SymbolState::Resolved); auto &MI = MaterializingInfos[Name]; for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { - Q->notifySymbolMetRequiredState(Name, Sym); + Q->notifySymbolMetRequiredState(Name, ResolvedSym); + Q->removeQueryDependence(*this, Name); if (Q->isComplete()) CompletedQueries.insert(std::move(Q)); } } - - return CompletedQueries; }); + assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) && + "Can't fail symbols and completed queries at the same time"); + + // If we failed any symbols then return an error. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap)); + } + + // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { assert(Q->isComplete() && "Q not completed"); Q->handleComplete(); } + + return Error::success(); } -void JITDylib::emit(const SymbolFlagsMap &Emitted) { - auto CompletedQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet CompletedQueries; +Error JITDylib::emit(const SymbolFlagsMap &Emitted) { + AsynchronousSymbolQuerySet CompletedQueries; + SymbolNameSet SymbolsInErrorState; + ES.runSessionLocked([&, this]() { + std::vector<SymbolTable::iterator> Worklist; + + // Scan to build worklist, record any symbols in the erorr state. for (const auto &KV : Emitted) { - const auto &Name = KV.first; + auto &Name = KV.first; + + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "No symbol table entry for Name"); + + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(Name); + else + Worklist.push_back(SymI); + } + + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) + return; + + // Otherwise update dependencies and move to the emitted state. + while (!Worklist.empty()) { + auto SymI = Worklist.back(); + Worklist.pop_back(); + + auto &Name = SymI->first; + auto &SymEntry = SymI->second; + + // Move symbol to the emitted state. + assert(SymEntry.getState() == SymbolState::Resolved && + "Emitting from state other than Resolved"); + SymEntry.setState(SymbolState::Emitted); auto MII = MaterializingInfos.find(Name); assert(MII != MaterializingInfos.end() && "Missing MaterializingInfo entry"); - auto &MI = MII->second; // For each dependant, transfer this node's emitted dependencies to @@ -926,8 +1059,12 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { auto &DependantMI = DependantMII->second; // Remove the dependant's dependency on this node. + assert(DependantMI.UnemittedDependencies.count(this) && + "Dependant does not have an unemitted dependencies record for " + "this JITDylib"); assert(DependantMI.UnemittedDependencies[this].count(Name) && "Dependant does not count this symbol as a dependency?"); + DependantMI.UnemittedDependencies[this].erase(Name); if (DependantMI.UnemittedDependencies[this].empty()) DependantMI.UnemittedDependencies.erase(this); @@ -936,20 +1073,22 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { DependantJD.transferEmittedNodeDependencies(DependantMI, DependantName, MI); + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && + "Dependant has no entry in the Symbols table"); + auto &DependantSymEntry = DependantSymI->second; + // If the dependant is emitted and this node was the last of its // unemitted dependencies then the dependant node is now ready, so // notify any pending queries on the dependant node. - if (DependantMI.IsEmitted && + if (DependantSymEntry.getState() == SymbolState::Emitted && DependantMI.UnemittedDependencies.empty()) { assert(DependantMI.Dependants.empty() && "Dependants should be empty by now"); // Since this dependant is now ready, we erase its MaterializingInfo // and update its materializing state. - auto DependantSymI = DependantJD.Symbols.find(DependantName); - assert(DependantSymI != DependantJD.Symbols.end() && - "Dependant has no entry in the Symbols table"); - DependantSymI->second.setState(SymbolState::Ready); + DependantSymEntry.setState(SymbolState::Ready); for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState( @@ -963,12 +1102,9 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { } } } - MI.Dependants.clear(); - MI.IsEmitted = true; + MI.Dependants.clear(); if (MI.UnemittedDependencies.empty()) { - auto SymI = Symbols.find(Name); - assert(SymI != Symbols.end() && "Symbol has no entry in Symbols table"); SymI->second.setState(SymbolState::Ready); for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); @@ -979,80 +1115,138 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { MaterializingInfos.erase(MII); } } - - return CompletedQueries; }); + assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) && + "Can't fail symbols and completed queries at the same time"); + + // If we failed any symbols then return an error. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap)); + } + + // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { assert(Q->isComplete() && "Q is not complete"); Q->handleComplete(); } + + return Error::success(); } -void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { +void JITDylib::notifyFailed(FailedSymbolsWorklist Worklist) { + AsynchronousSymbolQuerySet FailedQueries; + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); - // FIXME: This should fail any transitively dependant symbols too. + // Failing no symbols is a no-op. + if (Worklist.empty()) + return; - auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet FailedQueries; - std::vector<MaterializingInfosMap::iterator> MIIsToRemove; + auto &ES = Worklist.front().first->getExecutionSession(); - for (auto &Name : FailedSymbols) { - auto I = Symbols.find(Name); - assert(I != Symbols.end() && "Symbol not present in this JITDylib"); - Symbols.erase(I); + ES.runSessionLocked([&]() { + while (!Worklist.empty()) { + assert(Worklist.back().first && "Failed JITDylib can not be null"); + auto &JD = *Worklist.back().first; + auto Name = std::move(Worklist.back().second); + Worklist.pop_back(); - auto MII = MaterializingInfos.find(Name); + (*FailedSymbolsMap)[&JD].insert(Name); + + assert(JD.Symbols.count(Name) && "No symbol table entry for Name"); + auto &Sym = JD.Symbols[Name]; - // If we have not created a MaterializingInfo for this symbol yet then - // there is nobody to notify. - if (MII == MaterializingInfos.end()) + // Move the symbol into the error state. + // Note that this may be redundant: The symbol might already have been + // moved to this state in response to the failure of a dependence. + Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError); + + // FIXME: Come up with a sane mapping of state to + // presence-of-MaterializingInfo so that we can assert presence / absence + // here, rather than testing it. + auto MII = JD.MaterializingInfos.find(Name); + + if (MII == JD.MaterializingInfos.end()) continue; - // Remove this symbol from the dependants list of any dependencies. - for (auto &KV : MII->second.UnemittedDependencies) { - auto *DependencyJD = KV.first; - auto &Dependencies = KV.second; - for (auto &DependencyName : Dependencies) { - auto DependencyMII = - DependencyJD->MaterializingInfos.find(DependencyName); - assert(DependencyMII != DependencyJD->MaterializingInfos.end() && - "Unemitted dependency must have a MaterializingInfo entry"); - assert(DependencyMII->second.Dependants.count(this) && - "Dependency's dependants list does not contain this JITDylib"); - assert(DependencyMII->second.Dependants[this].count(Name) && - "Dependency's dependants list does not contain dependant"); - DependencyMII->second.Dependants[this].erase(Name); + auto &MI = MII->second; + + // Move all dependants to the error state and disconnect from them. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + for (auto &DependantName : KV.second) { + assert(DependantJD.Symbols.count(DependantName) && + "No symbol table entry for DependantName"); + auto &DependantSym = DependantJD.Symbols[DependantName]; + DependantSym.setFlags(DependantSym.getFlags() | + JITSymbolFlags::HasError); + + assert(DependantJD.MaterializingInfos.count(DependantName) && + "No MaterializingInfo for dependant"); + auto &DependantMI = DependantJD.MaterializingInfos[DependantName]; + + auto UnemittedDepI = DependantMI.UnemittedDependencies.find(&JD); + assert(UnemittedDepI != DependantMI.UnemittedDependencies.end() && + "No UnemittedDependencies entry for this JITDylib"); + assert(UnemittedDepI->second.count(Name) && + "No UnemittedDependencies entry for this symbol"); + UnemittedDepI->second.erase(Name); + if (UnemittedDepI->second.empty()) + DependantMI.UnemittedDependencies.erase(UnemittedDepI); + + // If this symbol is already in the emitted state then we need to + // take responsibility for failing its queries, so add it to the + // worklist. + if (DependantSym.getState() == SymbolState::Emitted) { + assert(DependantMI.Dependants.empty() && + "Emitted symbol should not have dependants"); + Worklist.push_back(std::make_pair(&DependantJD, DependantName)); + } } } + MI.Dependants.clear(); - // Copy all the queries to the FailedQueries list, then abandon them. - // This has to be a copy, and the copy has to come before the abandon - // operation: Each Q.detach() call will reach back into this - // PendingQueries list to remove Q. - for (auto &Q : MII->second.pendingQueries()) - FailedQueries.insert(Q); - - MIIsToRemove.push_back(std::move(MII)); - } - - // Detach failed queries. - for (auto &Q : FailedQueries) - Q->detach(); + // Disconnect from all unemitted depenencies. + for (auto &KV : MI.UnemittedDependencies) { + auto &UnemittedDepJD = *KV.first; + for (auto &UnemittedDepName : KV.second) { + auto UnemittedDepMII = + UnemittedDepJD.MaterializingInfos.find(UnemittedDepName); + assert(UnemittedDepMII != UnemittedDepJD.MaterializingInfos.end() && + "Missing MII for unemitted dependency"); + assert(UnemittedDepMII->second.Dependants.count(&JD) && + "JD not listed as a dependant of unemitted dependency"); + assert(UnemittedDepMII->second.Dependants[&JD].count(Name) && + "Name is not listed as a dependant of unemitted dependency"); + UnemittedDepMII->second.Dependants[&JD].erase(Name); + if (UnemittedDepMII->second.Dependants[&JD].empty()) + UnemittedDepMII->second.Dependants.erase(&JD); + } + } + MI.UnemittedDependencies.clear(); - // Remove the MaterializingInfos. - for (auto &MII : MIIsToRemove) { - assert(!MII->second.hasQueriesPending() && - "Queries remain after symbol was failed"); + // Collect queries to be failed for this MII. + for (auto &Q : MII->second.pendingQueries()) { + // Add the query to the list to be failed and detach it. + FailedQueries.insert(Q); + Q->detach(); + } - MaterializingInfos.erase(MII); + assert(MI.Dependants.empty() && + "Can not delete MaterializingInfo with dependants still attached"); + assert(MI.UnemittedDependencies.empty() && + "Can not delete MaterializingInfo with unemitted dependencies " + "still attached"); + assert(!MI.hasQueriesPending() && + "Can not delete MaterializingInfo with queries pending"); + JD.MaterializingInfos.erase(MII); } - - return FailedQueries; }); - for (auto &Q : FailedQueriesToNotify) - Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); + for (auto &Q : FailedQueries) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbolsMap)); } void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, @@ -1159,10 +1353,18 @@ Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) { if (!Unresolved) return Unresolved.takeError(); - if (DefGenerator && !Unresolved->empty()) { - auto NewDefs = DefGenerator(*this, *Unresolved); + /// Run any definition generators. + for (auto &DG : DefGenerators) { + + // Bail out early if we've resolved everything. + if (Unresolved->empty()) + break; + + // Run this generator. + auto NewDefs = DG->tryToGenerate(*this, *Unresolved); if (!NewDefs) return NewDefs.takeError(); + if (!NewDefs->empty()) { auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs); if (!Unresolved2) @@ -1171,7 +1373,10 @@ Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) { assert(Unresolved2->empty() && "All fallback defs should have been found by lookupFlagsImpl"); } - }; + + for (auto &Name : *NewDefs) + Unresolved->erase(Name); + } return Result; }); } @@ -1197,15 +1402,34 @@ Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, MaterializationUnitList &MUs) { assert(Q && "Query can not be null"); - lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs); - if (DefGenerator && !Unresolved.empty()) { - auto NewDefs = DefGenerator(*this, Unresolved); + if (auto Err = lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs)) + return Err; + + // Run any definition generators. + for (auto &DG : DefGenerators) { + + // Bail out early if we have resolved everything. + if (Unresolved.empty()) + break; + + // Run the generator. + auto NewDefs = DG->tryToGenerate(*this, Unresolved); + + // If the generator returns an error then bail out. if (!NewDefs) return NewDefs.takeError(); + + // If the generator was able to generate new definitions for any of the + // unresolved symbols then lodge the query against them. if (!NewDefs->empty()) { for (auto &D : *NewDefs) Unresolved.erase(D); - lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs); + + // Lodge query. This can not fail as any new definitions were added + // by the generator under the session locked. Since they can't have + // started materializing yet the can not have failed. + cantFail(lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs)); + assert(NewDefs->empty() && "All fallback defs should have been found by lookupImpl"); } @@ -1214,7 +1438,7 @@ Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, return Error::success(); } -void JITDylib::lodgeQueryImpl( +Error JITDylib::lodgeQueryImpl( std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved, bool MatchNonExported, std::vector<std::unique_ptr<MaterializationUnit>> &MUs) { @@ -1235,6 +1459,14 @@ void JITDylib::lodgeQueryImpl( // Unresolved set. ToRemove.push_back(Name); + // If we matched against this symbol but it is in the error state then + // bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[this] = {Name}; + return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); + } + // If this symbol already meets the required state for then notify the // query and continue. if (SymI->second.getState() >= Q->getRequiredState()) { @@ -1277,6 +1509,8 @@ void JITDylib::lodgeQueryImpl( // Remove any symbols that we found. for (auto &Name : ToRemove) Unresolved.erase(Name); + + return Error::success(); } Expected<SymbolNameSet> @@ -1292,9 +1526,16 @@ JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, SymbolNameSet Unresolved = std::move(Names); auto Err = ES.runSessionLocked([&, this]() -> Error { QueryComplete = lookupImpl(Q, MUs, Unresolved); - if (DefGenerator && !Unresolved.empty()) { + + // Run any definition generators. + for (auto &DG : DefGenerators) { + + // Bail out early if we have resolved everything. + if (Unresolved.empty()) + break; + assert(!QueryComplete && "query complete but unresolved symbols remain?"); - auto NewDefs = DefGenerator(*this, Unresolved); + auto NewDefs = DG->tryToGenerate(*this, Unresolved); if (!NewDefs) return NewDefs.takeError(); if (!NewDefs->empty()) { @@ -1432,8 +1673,6 @@ void JITDylib::dump(raw_ostream &OS) { OS << " MaterializingInfos entries:\n"; for (auto &KV : MaterializingInfos) { OS << " \"" << *KV.first << "\":\n" - << " IsEmitted = " << (KV.second.IsEmitted ? "true" : "false") - << "\n" << " " << KV.second.pendingQueries().size() << " pending queries: { "; for (const auto &Q : KV.second.pendingQueries()) @@ -1486,13 +1725,6 @@ JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { return Result; } -JITDylib::AsynchronousSymbolQueryList -JITDylib::MaterializingInfo::takeAllQueries() { - AsynchronousSymbolQueryList Result; - std::swap(Result, PendingQueries); - return Result; -} - JITDylib::JITDylib(ExecutionSession &ES, std::string Name) : ES(ES), JITDylibName(std::move(Name)) { SearchOrder.push_back({this, true}); diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index f7fc5f8f1797..4a886ac0597c 100644 --- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -8,6 +8,7 @@ #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" @@ -67,7 +68,7 @@ CtorDtorIterator::Element CtorDtorIterator::operator*() const { } } - ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0)); + auto *Priority = cast<ConstantInt>(CS->getOperand(0)); Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr; if (Data && !isa<GlobalValue>(Data)) Data = nullptr; @@ -87,7 +88,7 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) { } void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) { - if (empty(CtorDtors)) + if (CtorDtors.empty()) return; MangleAndInterner Mangle( @@ -178,20 +179,20 @@ DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( : Dylib(std::move(Dylib)), Allow(std::move(Allow)), GlobalPrefix(GlobalPrefix) {} -Expected<DynamicLibrarySearchGenerator> +Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow) { std::string ErrMsg; auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); if (!Lib.isValid()) return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); - return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix, - std::move(Allow)); + return std::make_unique<DynamicLibrarySearchGenerator>( + std::move(Lib), GlobalPrefix, std::move(Allow)); } Expected<SymbolNameSet> -DynamicLibrarySearchGenerator::operator()(JITDylib &JD, - const SymbolNameSet &Names) { +DynamicLibrarySearchGenerator::tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolMap NewSymbols; @@ -226,5 +227,82 @@ DynamicLibrarySearchGenerator::operator()(JITDylib &JD, return Added; } +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName) { + auto ArchiveBuffer = errorOrToExpected(MemoryBuffer::getFile(FileName)); + + if (!ArchiveBuffer) + return ArchiveBuffer.takeError(); + + return Create(L, std::move(*ArchiveBuffer)); +} + +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Create( + ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer) { + Error Err = Error::success(); + + std::unique_ptr<StaticLibraryDefinitionGenerator> ADG( + new StaticLibraryDefinitionGenerator(L, std::move(ArchiveBuffer), Err)); + + if (Err) + return std::move(Err); + + return std::move(ADG); +} + +Expected<SymbolNameSet> +StaticLibraryDefinitionGenerator::tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) { + + DenseSet<std::pair<StringRef, StringRef>> ChildBufferInfos; + SymbolNameSet NewDefs; + + for (const auto &Name : Names) { + auto Child = Archive.findSym(*Name); + if (!Child) + return Child.takeError(); + if (*Child == None) + continue; + auto ChildBuffer = (*Child)->getMemoryBufferRef(); + if (!ChildBuffer) + return ChildBuffer.takeError(); + ChildBufferInfos.insert( + {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()}); + NewDefs.insert(Name); + } + + for (auto ChildBufferInfo : ChildBufferInfos) { + MemoryBufferRef ChildBufferRef(ChildBufferInfo.first, + ChildBufferInfo.second); + + if (auto Err = + L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef), VModuleKey())) + return std::move(Err); + + --UnrealizedObjects; + } + + return NewDefs; +} + +StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( + ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, Error &Err) + : L(L), ArchiveBuffer(std::move(ArchiveBuffer)), + Archive(*this->ArchiveBuffer, Err) { + + if (Err) + return; + + Error Err2 = Error::success(); + for (auto _ : Archive.children(Err2)) { + (void)_; + ++UnrealizedObjects; + } + + // No need to check this: We will leave it to the caller. + Err = std::move(Err2); +} + } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp index 81dfc02f55b2..d311f34179c7 100644 --- a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -22,9 +22,9 @@ void IRCompileLayer::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { void IRCompileLayer::emit(MaterializationResponsibility R, ThreadSafeModule TSM) { - assert(TSM.getModule() && "Module must not be null"); + assert(TSM && "Module must not be null"); - if (auto Obj = Compile(*TSM.getModule())) { + if (auto Obj = TSM.withModuleDo(Compile)) { { std::lock_guard<std::mutex> Lock(IRLayerMutex); if (NotifyCompiled) diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp index e3519284613e..845ecc71eb87 100644 --- a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -19,7 +19,7 @@ IRTransformLayer::IRTransformLayer(ExecutionSession &ES, void IRTransformLayer::emit(MaterializationResponsibility R, ThreadSafeModule TSM) { - assert(TSM.getModule() && "Module must not be null"); + assert(TSM && "Module must not be null"); if (auto TransformedTSM = Transform(std::move(TSM), R)) BaseLayer.emit(std::move(R), std::move(*TransformedTSM)); diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index cc3656fe5dc5..0295db7633dd 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -37,8 +37,9 @@ private: void materialize(MaterializationResponsibility R) override { SymbolMap Result; Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); - R.notifyResolved(Result); - R.notifyEmitted(); + // No dependencies, so these calls cannot fail. + cantFail(R.notifyResolved(Result)); + cantFail(R.notifyEmitted()); } void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { @@ -66,7 +67,7 @@ JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { std::lock_guard<std::mutex> Lock(CCMgrMutex); AddrToSymbol[*TrampolineAddr] = CallbackName; cantFail(CallbacksJD.define( - llvm::make_unique<CompileCallbackMaterializationUnit>( + std::make_unique<CompileCallbackMaterializationUnit>( std::move(CallbackName), std::move(Compile), ES.allocateVModule()))); return *TrampolineAddr; @@ -119,7 +120,8 @@ createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, return make_error<StringError>( std::string("No callback manager available for ") + T.str(), inconvertibleErrorCode()); - case Triple::aarch64: { + case Triple::aarch64: + case Triple::aarch64_32: { typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; return CCMgrT::Create(ES, ErrorHandlerAddress); } @@ -162,50 +164,51 @@ createLocalIndirectStubsManagerBuilder(const Triple &T) { switch (T.getArch()) { default: return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); }; case Triple::aarch64: + case Triple::aarch64_32: return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); }; case Triple::x86: return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcI386>>(); }; case Triple::mips: return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); }; case Triple::mipsel: return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); }; case Triple::mips64: case Triple::mips64el: return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcMips64>>(); }; case Triple::x86_64: if (T.getOS() == Triple::OSType::Win32) { return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); }; } else { return [](){ - return llvm::make_unique< + return std::make_unique< orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); }; } diff --git a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index df23547a9de3..1d3e6db913e2 100644 --- a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -8,6 +8,7 @@ #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/Support/Host.h" #include "llvm/Support/TargetRegistry.h" namespace llvm { @@ -22,7 +23,21 @@ JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { // FIXME: getProcessTriple is bogus. It returns the host LLVM was compiled on, // rather than a valid triple for the current process. - return JITTargetMachineBuilder(Triple(sys::getProcessTriple())); + JITTargetMachineBuilder TMBuilder((Triple(sys::getProcessTriple()))); + + // Retrieve host CPU name and sub-target features and add them to builder. + // Relocation model, code model and codegen opt level are kept to default + // values. + llvm::SubtargetFeatures SubtargetFeatures; + llvm::StringMap<bool> FeatureMap; + llvm::sys::getHostCPUFeatures(FeatureMap); + for (auto &Feature : FeatureMap) + SubtargetFeatures.AddFeature(Feature.first(), Feature.second); + + TMBuilder.setCPU(llvm::sys::getHostCPUName()); + TMBuilder.addFeatures(SubtargetFeatures.getFeatures()); + + return TMBuilder; } Expected<std::unique_ptr<TargetMachine>> diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp index b120691faf07..a80f78afe80f 100644 --- a/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -41,7 +41,8 @@ Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); - if (auto Err = applyDataLayout(*TSM.getModule())) + if (auto Err = + TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); })) return Err; return CompileLayer->add(JD, std::move(TSM), ES->allocateVModule()); @@ -63,12 +64,21 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { // If the config state provided an ObjectLinkingLayer factory then use it. if (S.CreateObjectLinkingLayer) - return S.CreateObjectLinkingLayer(ES); + return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple()); // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs // a new SectionMemoryManager for each object. - auto GetMemMgr = []() { return llvm::make_unique<SectionMemoryManager>(); }; - return llvm::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); + auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); }; + auto ObjLinkingLayer = + std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); + + if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) + ObjLinkingLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); + + // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence + // errors from some GCC / libstdc++ bots. Remove this conversion (i.e. + // just return ObjLinkingLayer) once those bots are upgraded. + return std::unique_ptr<ObjectLayer>(std::move(ObjLinkingLayer)); } Expected<IRCompileLayer::CompileFunction> @@ -92,7 +102,7 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : ES(S.ES ? std::move(S.ES) : llvm::make_unique<ExecutionSession>()), + : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main), DtorRunner(Main) { @@ -113,13 +123,13 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) Err = CompileFunction.takeError(); return; } - CompileLayer = llvm::make_unique<IRCompileLayer>( + CompileLayer = std::make_unique<IRCompileLayer>( *ES, *ObjLinkingLayer, std::move(*CompileFunction)); } if (S.NumCompileThreads > 0) { CompileLayer->setCloneToNewContextOnEmit(true); - CompileThreads = llvm::make_unique<ThreadPool>(S.NumCompileThreads); + CompileThreads = std::make_unique<ThreadPool>(S.NumCompileThreads); ES->setDispatchMaterialization( [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { // FIXME: Switch to move capture once we have c++14. @@ -166,10 +176,14 @@ Error LLLazyJITBuilderState::prepareForConstruction() { Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); - if (auto Err = applyDataLayout(*TSM.getModule())) - return Err; + if (auto Err = TSM.withModuleDo([&](Module &M) -> Error { + if (auto Err = applyDataLayout(M)) + return Err; - recordCtorDtors(*TSM.getModule()); + recordCtorDtors(M); + return Error::success(); + })) + return Err; return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); } @@ -212,10 +226,10 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { } // Create the transform layer. - TransformLayer = llvm::make_unique<IRTransformLayer>(*ES, *CompileLayer); + TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer); // Create the COD layer. - CODLayer = llvm::make_unique<CompileOnDemandLayer>( + CODLayer = std::make_unique<CompileOnDemandLayer>( *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); if (S.NumCompileThreads > 0) diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp index 3ed2dabf4545..580e2682ec8c 100644 --- a/lib/ExecutionEngine/Orc/Layer.cpp +++ b/lib/ExecutionEngine/Orc/Layer.cpp @@ -19,7 +19,7 @@ IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {} IRLayer::~IRLayer() {} Error IRLayer::add(JITDylib &JD, ThreadSafeModule TSM, VModuleKey K) { - return JD.define(llvm::make_unique<BasicIRLayerMaterializationUnit>( + return JD.define(std::make_unique<BasicIRLayerMaterializationUnit>( *this, std::move(K), std::move(TSM))); } @@ -29,15 +29,17 @@ IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, assert(this->TSM && "Module must not be null"); - MangleAndInterner Mangle(ES, this->TSM.getModule()->getDataLayout()); - for (auto &G : this->TSM.getModule()->global_values()) { - if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() && - !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) { - auto MangledName = Mangle(G.getName()); - SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); - SymbolToDefinition[MangledName] = &G; + MangleAndInterner Mangle(ES, this->TSM.getModuleUnlocked()->getDataLayout()); + this->TSM.withModuleDo([&](Module &M) { + for (auto &G : M.global_values()) { + if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() && + !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) { + auto MangledName = Mangle(G.getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); + SymbolToDefinition[MangledName] = &G; + } } - } + }); } IRMaterializationUnit::IRMaterializationUnit( @@ -47,8 +49,9 @@ IRMaterializationUnit::IRMaterializationUnit( TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} StringRef IRMaterializationUnit::getName() const { - if (TSM.getModule()) - return TSM.getModule()->getModuleIdentifier(); + if (TSM) + return TSM.withModuleDo( + [](const Module &M) -> StringRef { return M.getModuleIdentifier(); }); return "<null module>"; } @@ -90,7 +93,6 @@ void BasicIRLayerMaterializationUnit::materialize( auto &N = R.getTargetJITDylib().getName(); #endif // NDEBUG - auto Lock = TSM.getContextLock(); LLVM_DEBUG(ES.runSessionLocked( [&]() { dbgs() << "Emitting, for " << N << ", " << *this << "\n"; });); L.emit(std::move(R), std::move(TSM)); diff --git a/lib/ExecutionEngine/Orc/LazyReexports.cpp b/lib/ExecutionEngine/Orc/LazyReexports.cpp index fc8205845654..93aabd817d60 100644 --- a/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -50,7 +50,6 @@ LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { SourceJD = I->second.first; SymbolName = I->second.second; } - auto LookupResult = ES.lookup(JITDylibSearchList({{SourceJD, true}}), SymbolName); @@ -91,6 +90,7 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, inconvertibleErrorCode()); case Triple::aarch64: + case Triple::aarch64_32: return LocalLazyCallThroughManager::Create<OrcAArch64>(ES, ErrorHandlerAddr); @@ -121,7 +121,8 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, - JITDylib &SourceJD, SymbolAliasMap CallableAliases, VModuleKey K) + JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc, + VModuleKey K) : MaterializationUnit(extractFlags(CallableAliases), std::move(K)), LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), CallableAliases(std::move(CallableAliases)), @@ -129,7 +130,8 @@ LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( [&ISManager](JITDylib &JD, const SymbolStringPtr &SymbolName, JITTargetAddress ResolvedAddr) { return ISManager.updatePointer(*SymbolName, ResolvedAddr); - })) {} + })), + AliaseeTable(SrcJDLoc) {} StringRef LazyReexportsMaterializationUnit::getName() const { return "<Lazy Reexports>"; @@ -149,7 +151,7 @@ void LazyReexportsMaterializationUnit::materialize( if (!CallableAliases.empty()) R.replace(lazyReexports(LCTManager, ISManager, SourceJD, - std::move(CallableAliases))); + std::move(CallableAliases), AliaseeTable)); IndirectStubsManager::StubInitsMap StubInits; for (auto &Alias : RequestedAliases) { @@ -168,6 +170,9 @@ void LazyReexportsMaterializationUnit::materialize( std::make_pair(*CallThroughTrampoline, Alias.second.AliasFlags); } + if (AliaseeTable != nullptr && !RequestedAliases.empty()) + AliaseeTable->trackImpls(RequestedAliases, &SourceJD); + if (auto Err = ISManager.createStubs(StubInits)) { SourceJD.getExecutionSession().reportError(std::move(Err)); R.failMaterialization(); @@ -178,8 +183,9 @@ void LazyReexportsMaterializationUnit::materialize( for (auto &Alias : RequestedAliases) Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); - R.notifyResolved(Stubs); - R.notifyEmitted(); + // No registered dependencies, so these calls cannot fail. + cantFail(R.notifyResolved(Stubs)); + cantFail(R.notifyEmitted()); } void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp index ce6368b57a89..9f9a6730b2c3 100644 --- a/lib/ExecutionEngine/Orc/Legacy.cpp +++ b/lib/ExecutionEngine/Orc/Legacy.cpp @@ -23,7 +23,8 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, for (auto &S : Symbols) InternedSymbols.insert(ES.intern(S)); - auto OnResolvedWithUnwrap = [OnResolved](Expected<SymbolMap> InternedResult) { + auto OnResolvedWithUnwrap = [OnResolved = std::move(OnResolved)]( + Expected<SymbolMap> InternedResult) mutable { if (!InternedResult) { OnResolved(InternedResult.takeError()); return; @@ -36,7 +37,7 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, }; auto Q = std::make_shared<AsynchronousSymbolQuery>( - InternedSymbols, SymbolState::Resolved, OnResolvedWithUnwrap); + InternedSymbols, SymbolState::Resolved, std::move(OnResolvedWithUnwrap)); auto Unresolved = R.lookup(Q, InternedSymbols); if (Unresolved.empty()) { diff --git a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index def0b300eca1..874decb2ade0 100644 --- a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -29,6 +29,13 @@ public: std::unique_ptr<MemoryBuffer> ObjBuffer) : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + ~ObjectLinkingLayerJITLinkContext() { + // If there is an object buffer return function then use it to + // return ownership of the buffer. + if (Layer.ReturnObjectBuffer) + Layer.ReturnObjectBuffer(std::move(ObjBuffer)); + } + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } MemoryBufferRef getObjectBuffer() const override { @@ -41,7 +48,7 @@ public: } void lookup(const DenseSet<StringRef> &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation) override { + std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { JITDylibSearchList SearchOrder; MR.getTargetJITDylib().withSearchOrderDo( @@ -54,18 +61,16 @@ public: InternedSymbols.insert(ES.intern(S)); // OnResolve -- De-intern the symbols and pass the result to the linker. - // FIXME: Capture LookupContinuation by move once we have c++14. - auto SharedLookupContinuation = - std::make_shared<JITLinkAsyncLookupContinuation>( - std::move(LookupContinuation)); - auto OnResolve = [SharedLookupContinuation](Expected<SymbolMap> Result) { + auto OnResolve = [this, LookupContinuation = std::move(LC)]( + Expected<SymbolMap> Result) mutable { + auto Main = Layer.getExecutionSession().intern("_main"); if (!Result) - (*SharedLookupContinuation)(Result.takeError()); + LookupContinuation->run(Result.takeError()); else { AsyncLookupResult LR; for (auto &KV : *Result) LR[*KV.first] = KV.second; - (*SharedLookupContinuation)(std::move(LR)); + LookupContinuation->run(std::move(LR)); } }; @@ -75,29 +80,25 @@ public: }); } - void notifyResolved(AtomGraph &G) override { + void notifyResolved(LinkGraph &G) override { auto &ES = Layer.getExecutionSession(); SymbolFlagsMap ExtraSymbolsToClaim; bool AutoClaim = Layer.AutoClaimObjectSymbols; SymbolMap InternedResult; - for (auto *DA : G.defined_atoms()) - if (DA->hasName() && DA->isGlobal()) { - auto InternedName = ES.intern(DA->getName()); + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && Sym->getScope() != Scope::Local) { + auto InternedName = ES.intern(Sym->getName()); JITSymbolFlags Flags; - if (DA->isExported()) - Flags |= JITSymbolFlags::Exported; - if (DA->isWeak()) - Flags |= JITSymbolFlags::Weak; - if (DA->isCallable()) + if (Sym->isCallable()) Flags |= JITSymbolFlags::Callable; - if (DA->isCommon()) - Flags |= JITSymbolFlags::Common; + if (Sym->getScope() == Scope::Default) + Flags |= JITSymbolFlags::Exported; InternedResult[InternedName] = - JITEvaluatedSymbol(DA->getAddress(), Flags); + JITEvaluatedSymbol(Sym->getAddress(), Flags); if (AutoClaim && !MR.getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -105,17 +106,17 @@ public: } } - for (auto *A : G.absolute_atoms()) - if (A->hasName()) { - auto InternedName = ES.intern(A->getName()); + for (auto *Sym : G.absolute_symbols()) + if (Sym->hasName()) { + auto InternedName = ES.intern(Sym->getName()); JITSymbolFlags Flags; Flags |= JITSymbolFlags::Absolute; - if (A->isWeak()) - Flags |= JITSymbolFlags::Weak; - if (A->isCallable()) + if (Sym->isCallable()) Flags |= JITSymbolFlags::Callable; + if (Sym->getLinkage() == Linkage::Weak) + Flags |= JITSymbolFlags::Weak; InternedResult[InternedName] = - JITEvaluatedSymbol(A->getAddress(), Flags); + JITEvaluatedSymbol(Sym->getAddress(), Flags); if (AutoClaim && !MR.getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -126,35 +127,38 @@ public: if (!ExtraSymbolsToClaim.empty()) if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) return notifyFailed(std::move(Err)); - - MR.notifyResolved(InternedResult); - + if (auto Err = MR.notifyResolved(InternedResult)) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + return; + } Layer.notifyLoaded(MR); } void notifyFinalized( std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { - if (auto Err = Layer.notifyEmitted(MR, std::move(A))) { Layer.getExecutionSession().reportError(std::move(Err)); MR.failMaterialization(); - return; } - MR.notifyEmitted(); + if (auto Err = MR.notifyEmitted()) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + } } - AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { - return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; } Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { // Add passes to mark duplicate defs as should-discard, and to walk the - // atom graph to build the symbol dependence graph. + // link graph to build the symbol dependence graph. Config.PrePrunePasses.push_back( - [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + [this](LinkGraph &G) { return externalizeWeakAndCommonSymbols(G); }); Config.PostPrunePasses.push_back( - [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); }); Layer.modifyPassConfig(MR, TT, Config); @@ -162,65 +166,59 @@ public: } private: - using AnonAtomNamedDependenciesMap = - DenseMap<const DefinedAtom *, SymbolNameSet>; + using AnonToNamedDependenciesMap = DenseMap<const Symbol *, SymbolNameSet>; - Error markSymbolsToDiscard(AtomGraph &G) { + Error externalizeWeakAndCommonSymbols(LinkGraph &G) { auto &ES = Layer.getExecutionSession(); - for (auto *DA : G.defined_atoms()) - if (DA->isWeak() && DA->hasName()) { - auto S = ES.intern(DA->getName()); - auto I = MR.getSymbols().find(S); - if (I == MR.getSymbols().end()) - DA->setShouldDiscard(true); + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { + if (!MR.getSymbols().count(ES.intern(Sym->getName()))) + G.makeExternal(*Sym); } - for (auto *A : G.absolute_atoms()) - if (A->isWeak() && A->hasName()) { - auto S = ES.intern(A->getName()); - auto I = MR.getSymbols().find(S); - if (I == MR.getSymbols().end()) - A->setShouldDiscard(true); + for (auto *Sym : G.absolute_symbols()) + if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { + if (!MR.getSymbols().count(ES.intern(Sym->getName()))) + G.makeExternal(*Sym); } return Error::success(); } - Error markResponsibilitySymbolsLive(AtomGraph &G) const { + Error markResponsibilitySymbolsLive(LinkGraph &G) const { auto &ES = Layer.getExecutionSession(); - for (auto *DA : G.defined_atoms()) - if (DA->hasName() && - MR.getSymbols().count(ES.intern(DA->getName()))) - DA->setLive(true); + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && MR.getSymbols().count(ES.intern(Sym->getName()))) + Sym->setLive(true); return Error::success(); } - Error computeNamedSymbolDependencies(AtomGraph &G) { + Error computeNamedSymbolDependencies(LinkGraph &G) { auto &ES = MR.getTargetJITDylib().getExecutionSession(); auto AnonDeps = computeAnonDeps(G); - for (auto *DA : G.defined_atoms()) { + for (auto *Sym : G.defined_symbols()) { // Skip anonymous and non-global atoms: we do not need dependencies for // these. - if (!DA->hasName() || !DA->isGlobal()) + if (Sym->getScope() == Scope::Local) continue; - auto DAName = ES.intern(DA->getName()); - SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + auto SymName = ES.intern(Sym->getName()); + SymbolNameSet &SymDeps = NamedSymbolDeps[SymName]; - for (auto &E : DA->edges()) { - auto &TA = E.getTarget(); + for (auto &E : Sym->getBlock().edges()) { + auto &TargetSym = E.getTarget(); - if (TA.hasName()) - DADeps.insert(ES.intern(TA.getName())); + if (TargetSym.getScope() != Scope::Local) + SymDeps.insert(ES.intern(TargetSym.getName())); else { - assert(TA.isDefined() && "Anonymous atoms must be defined"); - auto &DTA = static_cast<DefinedAtom &>(TA); - auto I = AnonDeps.find(&DTA); + assert(TargetSym.isDefined() && + "Anonymous/local symbols must be defined"); + auto I = AnonDeps.find(&TargetSym); if (I != AnonDeps.end()) for (auto &S : I->second) - DADeps.insert(S); + SymDeps.insert(S); } } } @@ -228,58 +226,59 @@ private: return Error::success(); } - AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + AnonToNamedDependenciesMap computeAnonDeps(LinkGraph &G) { auto &ES = MR.getTargetJITDylib().getExecutionSession(); - AnonAtomNamedDependenciesMap DepMap; + AnonToNamedDependenciesMap DepMap; - // For all anonymous atoms: + // For all anonymous symbols: // (1) Add their named dependencies. // (2) Add them to the worklist for further iteration if they have any - // depend on any other anonymous atoms. + // depend on any other anonymous symbols. struct WorklistEntry { - WorklistEntry(DefinedAtom *DA, DenseSet<DefinedAtom *> DAAnonDeps) - : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + WorklistEntry(Symbol *Sym, DenseSet<Symbol *> SymAnonDeps) + : Sym(Sym), SymAnonDeps(std::move(SymAnonDeps)) {} - DefinedAtom *DA = nullptr; - DenseSet<DefinedAtom *> DAAnonDeps; + Symbol *Sym = nullptr; + DenseSet<Symbol *> SymAnonDeps; }; std::vector<WorklistEntry> Worklist; - for (auto *DA : G.defined_atoms()) - if (!DA->hasName()) { - auto &DANamedDeps = DepMap[DA]; - DenseSet<DefinedAtom *> DAAnonDeps; - - for (auto &E : DA->edges()) { - auto &TA = E.getTarget(); - if (TA.hasName()) - DANamedDeps.insert(ES.intern(TA.getName())); + for (auto *Sym : G.defined_symbols()) + if (!Sym->hasName()) { + auto &SymNamedDeps = DepMap[Sym]; + DenseSet<Symbol *> SymAnonDeps; + + for (auto &E : Sym->getBlock().edges()) { + auto &TargetSym = E.getTarget(); + if (TargetSym.hasName()) + SymNamedDeps.insert(ES.intern(TargetSym.getName())); else { - assert(TA.isDefined() && "Anonymous atoms must be defined"); - DAAnonDeps.insert(static_cast<DefinedAtom *>(&TA)); + assert(TargetSym.isDefined() && + "Anonymous symbols must be defined"); + SymAnonDeps.insert(&TargetSym); } } - if (!DAAnonDeps.empty()) - Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + if (!SymAnonDeps.empty()) + Worklist.push_back(WorklistEntry(Sym, std::move(SymAnonDeps))); } - // Loop over all anonymous atoms with anonymous dependencies, propagating + // Loop over all anonymous symbols with anonymous dependencies, propagating // their respective *named* dependencies. Iterate until we hit a stable // state. bool Changed; do { Changed = false; for (auto &WLEntry : Worklist) { - auto *DA = WLEntry.DA; - auto &DANamedDeps = DepMap[DA]; - auto &DAAnonDeps = WLEntry.DAAnonDeps; + auto *Sym = WLEntry.Sym; + auto &SymNamedDeps = DepMap[Sym]; + auto &SymAnonDeps = WLEntry.SymAnonDeps; - for (auto *TA : DAAnonDeps) { - auto I = DepMap.find(TA); + for (auto *TargetSym : SymAnonDeps) { + auto I = DepMap.find(TargetSym); if (I != DepMap.end()) for (const auto &S : I->second) - Changed |= DANamedDeps.insert(S).second; + Changed |= SymNamedDeps.insert(S).second; } } } while (Changed); @@ -330,7 +329,7 @@ ObjectLinkingLayer::~ObjectLinkingLayer() { void ObjectLinkingLayer::emit(MaterializationResponsibility R, std::unique_ptr<MemoryBuffer> O) { assert(O && "Object must not be null"); - jitLink(llvm::make_unique<ObjectLinkingLayerJITLinkContext>( + jitLink(std::make_unique<ObjectLinkingLayerJITLinkContext>( *this, std::move(R), std::move(O))); } @@ -410,7 +409,7 @@ Error ObjectLinkingLayer::removeAllModules() { } EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( - jitlink::EHFrameRegistrar &Registrar) + EHFrameRegistrar &Registrar) : Registrar(Registrar) {} void EHFrameRegistrationPlugin::modifyPassConfig( @@ -419,61 +418,66 @@ void EHFrameRegistrationPlugin::modifyPassConfig( assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?"); PassConfig.PostFixupPasses.push_back( - createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr) { + createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr, + size_t Size) { if (Addr) - InProcessLinks[&MR] = Addr; + InProcessLinks[&MR] = { Addr, Size }; })); } Error EHFrameRegistrationPlugin::notifyEmitted( MaterializationResponsibility &MR) { - auto EHFrameAddrItr = InProcessLinks.find(&MR); - if (EHFrameAddrItr == InProcessLinks.end()) + auto EHFrameRangeItr = InProcessLinks.find(&MR); + if (EHFrameRangeItr == InProcessLinks.end()) return Error::success(); - auto EHFrameAddr = EHFrameAddrItr->second; - assert(EHFrameAddr && "eh-frame addr to register can not be null"); + auto EHFrameRange = EHFrameRangeItr->second; + assert(EHFrameRange.Addr && + "eh-frame addr to register can not be null"); - InProcessLinks.erase(EHFrameAddrItr); + InProcessLinks.erase(EHFrameRangeItr); if (auto Key = MR.getVModuleKey()) - TrackedEHFrameAddrs[Key] = EHFrameAddr; + TrackedEHFrameRanges[Key] = EHFrameRange; else - UntrackedEHFrameAddrs.push_back(EHFrameAddr); + UntrackedEHFrameRanges.push_back(EHFrameRange); - return Registrar.registerEHFrames(EHFrameAddr); + return Registrar.registerEHFrames(EHFrameRange.Addr, EHFrameRange.Size); } Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { - auto EHFrameAddrItr = TrackedEHFrameAddrs.find(K); - if (EHFrameAddrItr == TrackedEHFrameAddrs.end()) + auto EHFrameRangeItr = TrackedEHFrameRanges.find(K); + if (EHFrameRangeItr == TrackedEHFrameRanges.end()) return Error::success(); - auto EHFrameAddr = EHFrameAddrItr->second; - assert(EHFrameAddr && "Tracked eh-frame addr must not be null"); + auto EHFrameRange = EHFrameRangeItr->second; + assert(EHFrameRange.Addr && "Tracked eh-frame range must not be null"); - TrackedEHFrameAddrs.erase(EHFrameAddrItr); + TrackedEHFrameRanges.erase(EHFrameRangeItr); - return Registrar.deregisterEHFrames(EHFrameAddr); + return Registrar.deregisterEHFrames(EHFrameRange.Addr, EHFrameRange.Size); } Error EHFrameRegistrationPlugin::notifyRemovingAllModules() { - std::vector<JITTargetAddress> EHFrameAddrs = std::move(UntrackedEHFrameAddrs); - EHFrameAddrs.reserve(EHFrameAddrs.size() + TrackedEHFrameAddrs.size()); + std::vector<EHFrameRange> EHFrameRanges = + std::move(UntrackedEHFrameRanges); + EHFrameRanges.reserve(EHFrameRanges.size() + TrackedEHFrameRanges.size()); - for (auto &KV : TrackedEHFrameAddrs) - EHFrameAddrs.push_back(KV.second); + for (auto &KV : TrackedEHFrameRanges) + EHFrameRanges.push_back(KV.second); - TrackedEHFrameAddrs.clear(); + TrackedEHFrameRanges.clear(); Error Err = Error::success(); - while (!EHFrameAddrs.empty()) { - auto EHFrameAddr = EHFrameAddrs.back(); - assert(EHFrameAddr && "Untracked eh-frame addr must not be null"); - EHFrameAddrs.pop_back(); - Err = joinErrors(std::move(Err), Registrar.deregisterEHFrames(EHFrameAddr)); + while (!EHFrameRanges.empty()) { + auto EHFrameRange = EHFrameRanges.back(); + assert(EHFrameRange.Addr && "Untracked eh-frame range must not be null"); + EHFrameRanges.pop_back(); + Err = joinErrors(std::move(Err), + Registrar.deregisterEHFrames(EHFrameRange.Addr, + EHFrameRange.Size)); } return Err; diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 98129e1690d2..e0af3df9d010 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -97,7 +97,7 @@ public: template <typename LayerT> std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) { - return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer); + return std::make_unique<GenericLayerImpl<LayerT>>(Layer); } } // end namespace detail @@ -316,7 +316,8 @@ public: if (auto Err = CtorRunner.runViaLayer(*this)) return std::move(Err); - IRStaticDestructorRunners.emplace_back(std::move(DtorNames), K); + IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation, + std::move(DtorNames), K); return K; } @@ -326,7 +327,7 @@ public: LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { return addIRModule(CompileLayer, std::move(M), - llvm::make_unique<SectionMemoryManager>(), + std::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } @@ -340,7 +341,7 @@ public: inconvertibleErrorCode()); return addIRModule(*CODLayer, std::move(M), - llvm::make_unique<SectionMemoryManager>(), + std::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } @@ -468,7 +469,7 @@ private: if (!CCMgr) return nullptr; - return llvm::make_unique<CODLayerT>( + return std::make_unique<CODLayerT>( AcknowledgeORCv1Deprecation, ES, CompileLayer, [&Resolvers](orc::VModuleKey K) { auto ResolverI = Resolvers.find(K); diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index b22ecd5f80a1..939cd539d1fb 100644 --- a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -27,9 +27,9 @@ public: // Build an OnResolve callback to unwrap the interned strings and pass them // to the OnResolved callback. - // FIXME: Switch to move capture of OnResolved once we have c++14. auto OnResolvedWithUnwrap = - [OnResolved](Expected<SymbolMap> InternedResult) { + [OnResolved = std::move(OnResolved)]( + Expected<SymbolMap> InternedResult) mutable { if (!InternedResult) { OnResolved(InternedResult.takeError()); return; @@ -50,7 +50,7 @@ public: MR.getTargetJITDylib().withSearchOrderDo( [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); ES.lookup(SearchOrder, InternedSymbols, SymbolState::Resolved, - OnResolvedWithUnwrap, RegisterDependencies); + std::move(OnResolvedWithUnwrap), RegisterDependencies); } Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) { @@ -133,8 +133,6 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, JITDylibSearchOrderResolver Resolver(*SharedR); - // FIXME: Switch to move-capture for the 'O' buffer once we have c++14. - MemoryBuffer *UnownedObjBuffer = O.release(); jitLinkForORC( **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections, [this, K, SharedR, &Obj, InternalSymbols]( @@ -143,9 +141,8 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo), ResolvedSymbols, *InternalSymbols); }, - [this, K, SharedR, UnownedObjBuffer](Error Err) { - std::unique_ptr<MemoryBuffer> ObjBuffer(UnownedObjBuffer); - onObjEmit(K, std::move(ObjBuffer), *SharedR, std::move(Err)); + [this, K, SharedR, O = std::move(O)](Error Err) mutable { + onObjEmit(K, std::move(O), *SharedR, std::move(Err)); }); } @@ -184,7 +181,10 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) return Err; - R.notifyResolved(Symbols); + if (auto Err = R.notifyResolved(Symbols)) { + R.failMaterialization(); + return Err; + } if (NotifyLoaded) NotifyLoaded(K, Obj, *LoadedObjInfo); @@ -201,7 +201,11 @@ void RTDyldObjectLinkingLayer::onObjEmit( return; } - R.notifyEmitted(); + if (auto Err = R.notifyEmitted()) { + getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + return; + } if (NotifyEmitted) NotifyEmitted(K, std::move(ObjBuffer)); diff --git a/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp new file mode 100644 index 000000000000..f22acf50419d --- /dev/null +++ b/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp @@ -0,0 +1,307 @@ +//===-- SpeculateAnalyses.cpp --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/SpeculateAnalyses.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/CFG.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/ErrorHandling.h" + +#include <algorithm> + +namespace { +using namespace llvm; +SmallVector<const BasicBlock *, 8> findBBwithCalls(const Function &F, + bool IndirectCall = false) { + SmallVector<const BasicBlock *, 8> BBs; + + auto findCallInst = [&IndirectCall](const Instruction &I) { + if (auto Call = dyn_cast<CallBase>(&I)) + return Call->isIndirectCall() ? IndirectCall : true; + else + return false; + }; + for (auto &BB : F) + if (findCallInst(*BB.getTerminator()) || + llvm::any_of(BB.instructionsWithoutDebug(), findCallInst)) + BBs.emplace_back(&BB); + + return BBs; +} +} // namespace + +// Implementations of Queries shouldn't need to lock the resources +// such as LLVMContext, each argument (function) has a non-shared LLVMContext +// Plus, if Queries contain states necessary locking scheme should be provided. +namespace llvm { +namespace orc { + +// Collect direct calls only +void SpeculateQuery::findCalles(const BasicBlock *BB, + DenseSet<StringRef> &CallesNames) { + assert(BB != nullptr && "Traversing Null BB to find calls?"); + + auto getCalledFunction = [&CallesNames](const CallBase *Call) { + auto CalledValue = Call->getCalledOperand()->stripPointerCasts(); + if (auto DirectCall = dyn_cast<Function>(CalledValue)) + CallesNames.insert(DirectCall->getName()); + }; + for (auto &I : BB->instructionsWithoutDebug()) + if (auto CI = dyn_cast<CallInst>(&I)) + getCalledFunction(CI); + + if (auto II = dyn_cast<InvokeInst>(BB->getTerminator())) + getCalledFunction(II); +} + +bool SpeculateQuery::isStraightLine(const Function &F) { + return llvm::all_of(F.getBasicBlockList(), [](const BasicBlock &BB) { + return BB.getSingleSuccessor() != nullptr; + }); +} + +// BlockFreqQuery Implementations + +size_t BlockFreqQuery::numBBToGet(size_t numBB) { + // small CFG + if (numBB < 4) + return numBB; + // mid-size CFG + else if (numBB < 20) + return (numBB / 2); + else + return (numBB / 2) + (numBB / 4); +} + +BlockFreqQuery::ResultTy BlockFreqQuery::operator()(Function &F) { + DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles; + DenseSet<StringRef> Calles; + SmallVector<std::pair<const BasicBlock *, uint64_t>, 8> BBFreqs; + + PassBuilder PB; + FunctionAnalysisManager FAM; + PB.registerFunctionAnalyses(FAM); + + auto IBBs = findBBwithCalls(F); + + if (IBBs.empty()) + return None; + + auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F); + + for (const auto I : IBBs) + BBFreqs.push_back({I, BFI.getBlockFreq(I).getFrequency()}); + + assert(IBBs.size() == BBFreqs.size() && "BB Count Mismatch"); + + llvm::sort(BBFreqs.begin(), BBFreqs.end(), + [](decltype(BBFreqs)::const_reference BBF, + decltype(BBFreqs)::const_reference BBS) { + return BBF.second > BBS.second ? true : false; + }); + + // ignoring number of direct calls in a BB + auto Topk = numBBToGet(BBFreqs.size()); + + for (size_t i = 0; i < Topk; i++) + findCalles(BBFreqs[i].first, Calles); + + assert(!Calles.empty() && "Running Analysis on Function with no calls?"); + + CallerAndCalles.insert({F.getName(), std::move(Calles)}); + + return CallerAndCalles; +} + +// SequenceBBQuery Implementation +std::size_t SequenceBBQuery::getHottestBlocks(std::size_t TotalBlocks) { + if (TotalBlocks == 1) + return TotalBlocks; + return TotalBlocks / 2; +} + +// FIXME : find good implementation. +SequenceBBQuery::BlockListTy +SequenceBBQuery::rearrangeBB(const Function &F, const BlockListTy &BBList) { + BlockListTy RearrangedBBSet; + + for (auto &Block : F.getBasicBlockList()) + if (llvm::is_contained(BBList, &Block)) + RearrangedBBSet.push_back(&Block); + + assert(RearrangedBBSet.size() == BBList.size() && + "BasicBlock missing while rearranging?"); + return RearrangedBBSet; +} + +void SequenceBBQuery::traverseToEntryBlock(const BasicBlock *AtBB, + const BlockListTy &CallerBlocks, + const BackEdgesInfoTy &BackEdgesInfo, + const BranchProbabilityInfo *BPI, + VisitedBlocksInfoTy &VisitedBlocks) { + auto Itr = VisitedBlocks.find(AtBB); + if (Itr != VisitedBlocks.end()) { // already visited. + if (!Itr->second.Upward) + return; + Itr->second.Upward = false; + } else { + // Create hint for newly discoverd blocks. + WalkDirection BlockHint; + BlockHint.Upward = false; + // FIXME: Expensive Check + if (llvm::is_contained(CallerBlocks, AtBB)) + BlockHint.CallerBlock = true; + VisitedBlocks.insert(std::make_pair(AtBB, BlockHint)); + } + + const_pred_iterator PIt = pred_begin(AtBB), EIt = pred_end(AtBB); + // Move this check to top, when we have code setup to launch speculative + // compiles for function in entry BB, this triggers the speculative compiles + // before running the program. + if (PIt == EIt) // No Preds. + return; + + DenseSet<const BasicBlock *> PredSkipNodes; + + // Since we are checking for predecessor's backedges, this Block + // occurs in second position. + for (auto &I : BackEdgesInfo) + if (I.second == AtBB) + PredSkipNodes.insert(I.first); + + // Skip predecessors which source of back-edges. + for (; PIt != EIt; ++PIt) + // checking EdgeHotness is cheaper + if (BPI->isEdgeHot(*PIt, AtBB) && !PredSkipNodes.count(*PIt)) + traverseToEntryBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); +} + +void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB, + const BlockListTy &CallerBlocks, + const BackEdgesInfoTy &BackEdgesInfo, + const BranchProbabilityInfo *BPI, + VisitedBlocksInfoTy &VisitedBlocks) { + auto Itr = VisitedBlocks.find(AtBB); + if (Itr != VisitedBlocks.end()) { // already visited. + if (!Itr->second.Downward) + return; + Itr->second.Downward = false; + } else { + // Create hint for newly discoverd blocks. + WalkDirection BlockHint; + BlockHint.Downward = false; + // FIXME: Expensive Check + if (llvm::is_contained(CallerBlocks, AtBB)) + BlockHint.CallerBlock = true; + VisitedBlocks.insert(std::make_pair(AtBB, BlockHint)); + } + + succ_const_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB); + if (PIt == EIt) // No succs. + return; + + // If there are hot edges, then compute SuccSkipNodes. + DenseSet<const BasicBlock *> SuccSkipNodes; + + // Since we are checking for successor's backedges, this Block + // occurs in first position. + for (auto &I : BackEdgesInfo) + if (I.first == AtBB) + SuccSkipNodes.insert(I.second); + + for (; PIt != EIt; ++PIt) + if (BPI->isEdgeHot(AtBB, *PIt) && !SuccSkipNodes.count(*PIt)) + traverseToExitBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); +} + +// Get Block frequencies for blocks and take most frquently executed block, +// walk towards the entry block from those blocks and discover the basic blocks +// with call. +SequenceBBQuery::BlockListTy +SequenceBBQuery::queryCFG(Function &F, const BlockListTy &CallerBlocks) { + + BlockFreqInfoTy BBFreqs; + VisitedBlocksInfoTy VisitedBlocks; + BackEdgesInfoTy BackEdgesInfo; + + PassBuilder PB; + FunctionAnalysisManager FAM; + PB.registerFunctionAnalyses(FAM); + + auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F); + + llvm::FindFunctionBackedges(F, BackEdgesInfo); + + for (const auto I : CallerBlocks) + BBFreqs.push_back({I, BFI.getBlockFreq(I).getFrequency()}); + + llvm::sort(BBFreqs, [](decltype(BBFreqs)::const_reference Bbf, + decltype(BBFreqs)::const_reference Bbs) { + return Bbf.second > Bbs.second; + }); + + ArrayRef<std::pair<const BasicBlock *, uint64_t>> HotBlocksRef(BBFreqs); + HotBlocksRef = + HotBlocksRef.drop_back(BBFreqs.size() - getHottestBlocks(BBFreqs.size())); + + BranchProbabilityInfo *BPI = + FAM.getCachedResult<BranchProbabilityAnalysis>(F); + + // visit NHotBlocks, + // traverse upwards to entry + // traverse downwards to end. + + for (auto I : HotBlocksRef) { + traverseToEntryBlock(I.first, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); + traverseToExitBlock(I.first, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); + } + + BlockListTy MinCallerBlocks; + for (auto &I : VisitedBlocks) + if (I.second.CallerBlock) + MinCallerBlocks.push_back(std::move(I.first)); + + return rearrangeBB(F, MinCallerBlocks); +} + +SpeculateQuery::ResultTy SequenceBBQuery::operator()(Function &F) { + // reduce the number of lists! + DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles; + DenseSet<StringRef> Calles; + BlockListTy SequencedBlocks; + BlockListTy CallerBlocks; + + CallerBlocks = findBBwithCalls(F); + if (CallerBlocks.empty()) + return None; + + if (isStraightLine(F)) + SequencedBlocks = rearrangeBB(F, CallerBlocks); + else + SequencedBlocks = queryCFG(F, CallerBlocks); + + for (auto BB : SequencedBlocks) + findCalles(BB, Calles); + + CallerAndCalles.insert({F.getName(), std::move(Calles)}); + return CallerAndCalles; +} + +} // namespace orc +} // namespace llvm diff --git a/lib/ExecutionEngine/Orc/Speculation.cpp b/lib/ExecutionEngine/Orc/Speculation.cpp new file mode 100644 index 000000000000..f29201c147a1 --- /dev/null +++ b/lib/ExecutionEngine/Orc/Speculation.cpp @@ -0,0 +1,146 @@ +//===---------- speculation.cpp - Utilities for Speculation ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Speculation.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Support/Debug.h" + +#include <vector> + +namespace llvm { + +namespace orc { + +// ImplSymbolMap methods +void ImplSymbolMap::trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD) { + assert(SrcJD && "Tracking on Null Source .impl dylib"); + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + for (auto &I : ImplMaps) { + auto It = Maps.insert({I.first, {I.second.Aliasee, SrcJD}}); + // check rationale when independent dylibs have same symbol name? + assert(It.second && "ImplSymbols are already tracked for this Symbol?"); + (void)(It); + } +} + +// Trigger Speculative Compiles. +void Speculator::speculateForEntryPoint(Speculator *Ptr, uint64_t StubId) { + assert(Ptr && " Null Address Received in orc_speculate_for "); + Ptr->speculateFor(StubId); +} + +Error Speculator::addSpeculationRuntime(JITDylib &JD, + MangleAndInterner &Mangle) { + JITEvaluatedSymbol ThisPtr(pointerToJITTargetAddress(this), + JITSymbolFlags::Exported); + JITEvaluatedSymbol SpeculateForEntryPtr( + pointerToJITTargetAddress(&speculateForEntryPoint), + JITSymbolFlags::Exported); + return JD.define(absoluteSymbols({ + {Mangle("__orc_speculator"), ThisPtr}, // Data Symbol + {Mangle("__orc_speculate_for"), SpeculateForEntryPtr} // Callable Symbol + })); +} + +// If two modules, share the same LLVMContext, different threads must +// not access them concurrently without locking the associated LLVMContext +// this implementation follows this contract. +void IRSpeculationLayer::emit(MaterializationResponsibility R, + ThreadSafeModule TSM) { + + assert(TSM && "Speculation Layer received Null Module ?"); + assert(TSM.getContext().getContext() != nullptr && + "Module with null LLVMContext?"); + + // Instrumentation of runtime calls, lock the Module + TSM.withModuleDo([this, &R](Module &M) { + auto &MContext = M.getContext(); + auto SpeculatorVTy = StructType::create(MContext, "Class.Speculator"); + auto RuntimeCallTy = FunctionType::get( + Type::getVoidTy(MContext), + {SpeculatorVTy->getPointerTo(), Type::getInt64Ty(MContext)}, false); + auto RuntimeCall = + Function::Create(RuntimeCallTy, Function::LinkageTypes::ExternalLinkage, + "__orc_speculate_for", &M); + auto SpeclAddr = new GlobalVariable( + M, SpeculatorVTy, false, GlobalValue::LinkageTypes::ExternalLinkage, + nullptr, "__orc_speculator"); + + IRBuilder<> Mutator(MContext); + + // QueryAnalysis allowed to transform the IR source, one such example is + // Simplify CFG helps the static branch prediction heuristics! + for (auto &Fn : M.getFunctionList()) { + if (!Fn.isDeclaration()) { + + auto IRNames = QueryAnalysis(Fn); + // Instrument and register if Query has result + if (IRNames.hasValue()) { + + // Emit globals for each function. + auto LoadValueTy = Type::getInt8Ty(MContext); + auto SpeculatorGuard = new GlobalVariable( + M, LoadValueTy, false, GlobalValue::LinkageTypes::InternalLinkage, + ConstantInt::get(LoadValueTy, 0), + "__orc_speculate.guard.for." + Fn.getName()); + SpeculatorGuard->setAlignment(Align::None()); + SpeculatorGuard->setUnnamedAddr(GlobalValue::UnnamedAddr::Local); + + BasicBlock &ProgramEntry = Fn.getEntryBlock(); + // Create BasicBlocks before the program's entry basicblock + BasicBlock *SpeculateBlock = BasicBlock::Create( + MContext, "__orc_speculate.block", &Fn, &ProgramEntry); + BasicBlock *SpeculateDecisionBlock = BasicBlock::Create( + MContext, "__orc_speculate.decision.block", &Fn, SpeculateBlock); + + assert(SpeculateDecisionBlock == &Fn.getEntryBlock() && + "SpeculateDecisionBlock not updated?"); + Mutator.SetInsertPoint(SpeculateDecisionBlock); + + auto LoadGuard = + Mutator.CreateLoad(LoadValueTy, SpeculatorGuard, "guard.value"); + // if just loaded value equal to 0,return true. + auto CanSpeculate = + Mutator.CreateICmpEQ(LoadGuard, ConstantInt::get(LoadValueTy, 0), + "compare.to.speculate"); + Mutator.CreateCondBr(CanSpeculate, SpeculateBlock, &ProgramEntry); + + Mutator.SetInsertPoint(SpeculateBlock); + auto ImplAddrToUint = + Mutator.CreatePtrToInt(&Fn, Type::getInt64Ty(MContext)); + Mutator.CreateCall(RuntimeCallTy, RuntimeCall, + {SpeclAddr, ImplAddrToUint}); + Mutator.CreateStore(ConstantInt::get(LoadValueTy, 1), + SpeculatorGuard); + Mutator.CreateBr(&ProgramEntry); + + assert(Mutator.GetInsertBlock()->getParent() == &Fn && + "IR builder association mismatch?"); + S.registerSymbols(internToJITSymbols(IRNames.getValue()), + &R.getTargetJITDylib()); + } + } + } + }); + + assert(!TSM.withModuleDo([](const Module &M) { return verifyModule(M); }) && + "Speculation Instrumentation breaks IR?"); + + NextLayer.emit(std::move(R), std::move(TSM)); +} + +} // namespace orc +} // namespace llvm diff --git a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp index 4cb7376758a7..1f4e6f132115 100644 --- a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp +++ b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -23,41 +23,41 @@ ThreadSafeModule cloneToNewContext(ThreadSafeModule &TSM, if (!ShouldCloneDef) ShouldCloneDef = [](const GlobalValue &) { return true; }; - auto Lock = TSM.getContextLock(); + return TSM.withModuleDo([&](Module &M) { + SmallVector<char, 1> ClonedModuleBuffer; - SmallVector<char, 1> ClonedModuleBuffer; + { + std::set<GlobalValue *> ClonedDefsInSrc; + ValueToValueMapTy VMap; + auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) { + if (ShouldCloneDef(*GV)) { + ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); + return true; + } + return false; + }); - { - std::set<GlobalValue *> ClonedDefsInSrc; - ValueToValueMapTy VMap; - auto Tmp = CloneModule(*TSM.getModule(), VMap, [&](const GlobalValue *GV) { - if (ShouldCloneDef(*GV)) { - ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); - return true; - } - return false; - }); + if (UpdateClonedDefSource) + for (auto *GV : ClonedDefsInSrc) + UpdateClonedDefSource(*GV); - if (UpdateClonedDefSource) - for (auto *GV : ClonedDefsInSrc) - UpdateClonedDefSource(*GV); + BitcodeWriter BCWriter(ClonedModuleBuffer); - BitcodeWriter BCWriter(ClonedModuleBuffer); + BCWriter.writeModule(*Tmp); + BCWriter.writeSymtab(); + BCWriter.writeStrtab(); + } - BCWriter.writeModule(*Tmp); - BCWriter.writeSymtab(); - BCWriter.writeStrtab(); - } + MemoryBufferRef ClonedModuleBufferRef( + StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), + "cloned module buffer"); + ThreadSafeContext NewTSCtx(std::make_unique<LLVMContext>()); - MemoryBufferRef ClonedModuleBufferRef( - StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), - "cloned module buffer"); - ThreadSafeContext NewTSCtx(llvm::make_unique<LLVMContext>()); - - auto ClonedModule = - cantFail(parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext())); - ClonedModule->setModuleIdentifier(TSM.getModule()->getName()); - return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx)); + auto ClonedModule = cantFail( + parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext())); + ClonedModule->setModuleIdentifier(M.getName()); + return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx)); + }); } } // end namespace orc diff --git a/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index 5606421a3cb0..184388dc4d7a 100644 --- a/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -26,11 +26,11 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Mutex.h" -#include "llvm/Support/MutexGuard.h" #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" #include "llvm/Support/Threading.h" #include "llvm/Support/raw_ostream.h" +#include <mutex> #include <sys/mman.h> // mmap() #include <sys/types.h> // getpid() @@ -203,7 +203,7 @@ PerfJITEventListener::PerfJITEventListener() : Pid(::getpid()) { return; } - Dumpstream = make_unique<raw_fd_ostream>(DumpFd, true); + Dumpstream = std::make_unique<raw_fd_ostream>(DumpFd, true); LLVMPerfJitHeader Header = {0}; if (!FillMachine(Header)) @@ -420,7 +420,7 @@ void PerfJITEventListener::NotifyCode(Expected<llvm::StringRef> &Symbol, rec.Tid = get_threadid(); // avoid interspersing output - MutexGuard Guard(Mutex); + std::lock_guard<sys::Mutex> Guard(Mutex); rec.CodeIndex = CodeGeneration++; // under lock! @@ -462,7 +462,7 @@ void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr, // * char name[n] : source file name in ASCII, including null termination // avoid interspersing output - MutexGuard Guard(Mutex); + std::lock_guard<sys::Mutex> Guard(Mutex); Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec)); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index e26e6ce45db4..2df71a5e5e74 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -17,10 +17,11 @@ #include "RuntimeDyldMachO.h" #include "llvm/Object/COFF.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/MSVCErrorWorkarounds.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/MutexGuard.h" +#include <mutex> #include <future> @@ -120,7 +121,7 @@ static void dumpSectionMemory(const SectionEntry &S, StringRef State) { // Resolve the relocations for all symbols we currently know about. void RuntimeDyldImpl::resolveRelocations() { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // Print out the sections prior to relocation. LLVM_DEBUG(for (int i = 0, e = Sections.size(); i != e; ++i) @@ -156,7 +157,7 @@ void RuntimeDyldImpl::resolveLocalRelocations() { void RuntimeDyldImpl::mapSectionAddress(const void *LocalAddress, uint64_t TargetAddress) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); for (unsigned i = 0, e = Sections.size(); i != e; ++i) { if (Sections[i].getAddress() == LocalAddress) { reassignSectionAddress(i, TargetAddress); @@ -177,7 +178,7 @@ static Error getOffset(const SymbolRef &Sym, SectionRef Sec, Expected<RuntimeDyldImpl::ObjSectionToIDMap> RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { - MutexGuard locked(lock); + std::lock_guard<sys::Mutex> locked(lock); // Save information about our target Arch = (Triple::ArchType)Obj.getArch(); @@ -347,8 +348,12 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); SI != SE; ++SI) { StubMap Stubs; - section_iterator RelocatedSection = SI->getRelocatedSection(); + Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection(); + if (!RelSecOrErr) + return RelSecOrErr.takeError(); + + section_iterator RelocatedSection = *RelSecOrErr; if (RelocatedSection == SE) continue; @@ -535,9 +540,10 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, bool IsCode = Section.isText(); bool IsReadOnly = isReadOnlyData(Section); - StringRef Name; - if (auto EC = Section.getName(Name)) - return errorCodeToError(EC); + Expected<StringRef> NameOrErr = Section.getName(); + if (!NameOrErr) + return NameOrErr.takeError(); + StringRef Name = *NameOrErr; uint64_t StubBufSize = computeSectionStubBufSize(Obj, Section); @@ -646,7 +652,12 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(const ObjectFile &Obj, unsigned StubBufSize = 0; for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); SI != SE; ++SI) { - section_iterator RelSecI = SI->getRelocatedSection(); + + Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection(); + if (!RelSecOrErr) + report_fatal_error(toString(RelSecOrErr.takeError())); + + section_iterator RelSecI = *RelSecOrErr; if (!(RelSecI == Section)) continue; @@ -727,16 +738,17 @@ Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, // Assign the address of each symbol for (auto &Sym : SymbolsToAllocate) { - uint32_t Align = Sym.getAlignment(); + uint32_t Alignment = Sym.getAlignment(); uint64_t Size = Sym.getCommonSize(); StringRef Name; if (auto NameOrErr = Sym.getName()) Name = *NameOrErr; else return NameOrErr.takeError(); - if (Align) { + if (Alignment) { // This symbol has an alignment requirement. - uint64_t AlignOffset = OffsetToAlignment((uint64_t)Addr, Align); + uint64_t AlignOffset = + offsetToAlignment((uint64_t)Addr, Align(Alignment)); Addr += AlignOffset; Offset += AlignOffset; } @@ -777,9 +789,10 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, // anyway, so we should guarantee that the alignment is always at least 1. Alignment = std::max(1u, Alignment); - StringRef Name; - if (auto EC = Section.getName(Name)) - return errorCodeToError(EC); + Expected<StringRef> NameOrErr = Section.getName(); + if (!NameOrErr) + return NameOrErr.takeError(); + StringRef Name = *NameOrErr; StubBufSize = computeSectionStubBufSize(Obj, Section); @@ -917,7 +930,8 @@ void RuntimeDyldImpl::addRelocationForSymbol(const RelocationEntry &RE, uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr, unsigned AbiVariant) { - if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) { + if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be || + Arch == Triple::aarch64_32) { // This stub has to be able to access the full address space, // since symbol lookup won't necessarily find a handy, in-range, // PLT stub for functions which could be anywhere. @@ -1175,17 +1189,15 @@ Error RuntimeDyldImpl::resolveExternalSymbols() { } void RuntimeDyldImpl::finalizeAsync( - std::unique_ptr<RuntimeDyldImpl> This, std::function<void(Error)> OnEmitted, + std::unique_ptr<RuntimeDyldImpl> This, + unique_function<void(Error)> OnEmitted, std::unique_ptr<MemoryBuffer> UnderlyingBuffer) { - // FIXME: Move-capture OnRelocsApplied and UnderlyingBuffer once we have - // c++14. - auto SharedUnderlyingBuffer = - std::shared_ptr<MemoryBuffer>(std::move(UnderlyingBuffer)); auto SharedThis = std::shared_ptr<RuntimeDyldImpl>(std::move(This)); auto PostResolveContinuation = - [SharedThis, OnEmitted, SharedUnderlyingBuffer]( - Expected<JITSymbolResolver::LookupResult> Result) { + [SharedThis, OnEmitted = std::move(OnEmitted), + UnderlyingBuffer = std::move(UnderlyingBuffer)]( + Expected<JITSymbolResolver::LookupResult> Result) mutable { if (!Result) { OnEmitted(Result.takeError()); return; @@ -1219,7 +1231,7 @@ void RuntimeDyldImpl::finalizeAsync( } if (!Symbols.empty()) { - SharedThis->Resolver.lookup(Symbols, PostResolveContinuation); + SharedThis->Resolver.lookup(Symbols, std::move(PostResolveContinuation)); } else PostResolveContinuation(std::map<StringRef, JITEvaluatedSymbol>()); } @@ -1395,11 +1407,11 @@ void jitLinkForORC(object::ObjectFile &Obj, std::unique_ptr<MemoryBuffer> UnderlyingBuffer, RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, bool ProcessAllSections, - std::function<Error( + unique_function<Error( std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObj, std::map<StringRef, JITEvaluatedSymbol>)> OnLoaded, - std::function<void(Error)> OnEmitted) { + unique_function<void(Error)> OnEmitted) { RuntimeDyld RTDyld(MemMgr, Resolver); RTDyld.setProcessAllSections(ProcessAllSections); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index d4e3b0ba7670..27a7690db34f 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -50,18 +50,18 @@ llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch, switch (Arch) { default: llvm_unreachable("Unsupported target for RuntimeDyldCOFF."); case Triple::x86: - return make_unique<RuntimeDyldCOFFI386>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldCOFFI386>(MemMgr, Resolver); case Triple::thumb: - return make_unique<RuntimeDyldCOFFThumb>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldCOFFThumb>(MemMgr, Resolver); case Triple::x86_64: - return make_unique<RuntimeDyldCOFFX86_64>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldCOFFX86_64>(MemMgr, Resolver); } } std::unique_ptr<RuntimeDyld::LoadedObjectInfo> RuntimeDyldCOFF::loadObject(const object::ObjectFile &O) { if (auto ObjSectionToIDOrErr = loadObjectImpl(O)) { - return llvm::make_unique<LoadedCOFFObjectInfo>(*this, *ObjSectionToIDOrErr); + return std::make_unique<LoadedCOFFObjectInfo>(*this, *ObjSectionToIDOrErr); } else { HasError = true; raw_string_ostream ErrStream(ErrorStr); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index ec31ea4e573c..b9c5a12e08d8 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -851,7 +851,7 @@ RuntimeDyldChecker::RuntimeDyldChecker( GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, raw_ostream &ErrStream) - : Impl(::llvm::make_unique<RuntimeDyldCheckerImpl>( + : Impl(::std::make_unique<RuntimeDyldCheckerImpl>( std::move(IsSymbolValid), std::move(GetSymbolInfo), std::move(GetSectionInfo), std::move(GetStubInfo), std::move(GetGOTInfo), Endianness, Disassembler, InstPrinter, diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 60041a45e2b8..440ab4174a56 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -160,9 +160,13 @@ createRTDyldELFObject(MemoryBufferRef Buffer, const ObjectFile &SourceObject, // Iterate over all sections in the object. auto SI = SourceObject.section_begin(); for (const auto &Sec : Obj->sections()) { - StringRef SectionName; - Sec.getName(SectionName); - if (SectionName != "") { + Expected<StringRef> NameOrErr = Sec.getName(); + if (!NameOrErr) { + consumeError(NameOrErr.takeError()); + continue; + } + + if (*NameOrErr != "") { DataRefImpl ShdrRef = Sec.getRawDataRefImpl(); Elf_Shdr *shdr = const_cast<Elf_Shdr *>( reinterpret_cast<const Elf_Shdr *>(ShdrRef.p)); @@ -238,19 +242,19 @@ llvm::RuntimeDyldELF::create(Triple::ArchType Arch, JITSymbolResolver &Resolver) { switch (Arch) { default: - return make_unique<RuntimeDyldELF>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldELF>(MemMgr, Resolver); case Triple::mips: case Triple::mipsel: case Triple::mips64: case Triple::mips64el: - return make_unique<RuntimeDyldELFMips>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldELFMips>(MemMgr, Resolver); } } std::unique_ptr<RuntimeDyld::LoadedObjectInfo> RuntimeDyldELF::loadObject(const object::ObjectFile &O) { if (auto ObjSectionToIDOrErr = loadObjectImpl(O)) - return llvm::make_unique<LoadedELFObjectInfo>(*this, *ObjSectionToIDOrErr); + return std::make_unique<LoadedELFObjectInfo>(*this, *ObjSectionToIDOrErr); else { HasError = true; raw_string_ostream ErrStream(ErrorStr); @@ -567,10 +571,11 @@ Error RuntimeDyldELF::findPPC64TOCSection(const ELFObjectFileBase &Obj, // The TOC consists of sections .got, .toc, .tocbss, .plt in that // order. The TOC starts where the first of these sections starts. - for (auto &Section: Obj.sections()) { - StringRef SectionName; - if (auto EC = Section.getName(SectionName)) - return errorCodeToError(EC); + for (auto &Section : Obj.sections()) { + Expected<StringRef> NameOrErr = Section.getName(); + if (!NameOrErr) + return NameOrErr.takeError(); + StringRef SectionName = *NameOrErr; if (SectionName == ".got" || SectionName == ".toc" @@ -601,13 +606,19 @@ Error RuntimeDyldELF::findOPDEntrySection(const ELFObjectFileBase &Obj, // .opd entries for (section_iterator si = Obj.section_begin(), se = Obj.section_end(); si != se; ++si) { - section_iterator RelSecI = si->getRelocatedSection(); + + Expected<section_iterator> RelSecOrErr = si->getRelocatedSection(); + if (!RelSecOrErr) + report_fatal_error(toString(RelSecOrErr.takeError())); + + section_iterator RelSecI = *RelSecOrErr; if (RelSecI == Obj.section_end()) continue; - StringRef RelSectionName; - if (auto EC = RelSecI->getName(RelSectionName)) - return errorCodeToError(EC); + Expected<StringRef> NameOrErr = RelSecI->getName(); + if (!NameOrErr) + return NameOrErr.takeError(); + StringRef RelSectionName = *NameOrErr; if (RelSectionName != ".opd") continue; @@ -1865,7 +1876,12 @@ Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); SI != SE; ++SI) { if (SI->relocation_begin() != SI->relocation_end()) { - section_iterator RelocatedSection = SI->getRelocatedSection(); + Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection(); + if (!RelSecOrErr) + return make_error<RuntimeDyldError>( + toString(RelSecOrErr.takeError())); + + section_iterator RelocatedSection = *RelSecOrErr; ObjSectionToIDMap::iterator i = SectionMap.find(*RelocatedSection); assert (i != SectionMap.end()); SectionToGOTMap[i->second] = GOTSectionID; @@ -1879,8 +1895,14 @@ Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap::iterator i, e; for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) { const SectionRef &Section = i->first; + StringRef Name; - Section.getName(Name); + Expected<StringRef> NameOrErr = Section.getName(); + if (NameOrErr) + Name = *NameOrErr; + else + consumeError(NameOrErr.takeError()); + if (Name == ".eh_frame") { UnregisteredEHFrameSections.push_back(i->second); break; diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 68b3468fbc9d..cec7b92b8c48 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -549,7 +549,7 @@ public: void resolveLocalRelocations(); static void finalizeAsync(std::unique_ptr<RuntimeDyldImpl> This, - std::function<void(Error)> OnEmitted, + unique_function<void(Error)> OnEmitted, std::unique_ptr<MemoryBuffer> UnderlyingBuffer); void reassignSectionAddress(unsigned SectionID, uint64_t Addr); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index 202c3ca1c507..9ca76602ea18 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -233,7 +233,10 @@ RuntimeDyldMachOCRTPBase<Impl>::finalizeLoad(const ObjectFile &Obj, for (const auto &Section : Obj.sections()) { StringRef Name; - Section.getName(Name); + if (Expected<StringRef> NameOrErr = Section.getName()) + Name = *NameOrErr; + else + consumeError(NameOrErr.takeError()); // Force emission of the __text, __eh_frame, and __gcc_except_tab sections // if they're present. Otherwise call down to the impl to handle other @@ -351,20 +354,22 @@ RuntimeDyldMachO::create(Triple::ArchType Arch, llvm_unreachable("Unsupported target for RuntimeDyldMachO."); break; case Triple::arm: - return make_unique<RuntimeDyldMachOARM>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldMachOARM>(MemMgr, Resolver); case Triple::aarch64: - return make_unique<RuntimeDyldMachOAArch64>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldMachOAArch64>(MemMgr, Resolver); + case Triple::aarch64_32: + return std::make_unique<RuntimeDyldMachOAArch64>(MemMgr, Resolver); case Triple::x86: - return make_unique<RuntimeDyldMachOI386>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldMachOI386>(MemMgr, Resolver); case Triple::x86_64: - return make_unique<RuntimeDyldMachOX86_64>(MemMgr, Resolver); + return std::make_unique<RuntimeDyldMachOX86_64>(MemMgr, Resolver); } } std::unique_ptr<RuntimeDyld::LoadedObjectInfo> RuntimeDyldMachO::loadObject(const object::ObjectFile &O) { if (auto ObjSectionToIDOrErr = loadObjectImpl(O)) - return llvm::make_unique<LoadedMachOObjectInfo>(*this, + return std::make_unique<LoadedMachOObjectInfo>(*this, *ObjSectionToIDOrErr); else { HasError = true; diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index d2d74534cf90..dc4af08583de 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -284,14 +284,14 @@ public: // Look for and record the EH frame section IDs. for (const auto &SectionPair : SectionMap) { const object::SectionRef &Section = SectionPair.first; - StringRef Name; - if (auto EC = Section.getName(Name)) - return errorCodeToError(EC); + Expected<StringRef> NameOrErr = Section.getName(); + if (!NameOrErr) + return NameOrErr.takeError(); // Note unwind info is stored in .pdata but often points to .xdata // with an IMAGE_REL_AMD64_ADDR32NB relocation. Using a memory manager // that keeps sections ordered in relation to __ImageBase is necessary. - if (Name == ".pdata") + if ((*NameOrErr) == ".pdata") UnregisteredEHFrameSections.push_back(SectionPair.second); } return Error::success(); diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index 3bec8b979f7d..a76958a9e2c2 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -289,7 +289,10 @@ public: Error finalizeSection(const ObjectFile &Obj, unsigned SectionID, const SectionRef &Section) { StringRef Name; - Section.getName(Name); + if (Expected<StringRef> NameOrErr = Section.getName()) + Name = *NameOrErr; + else + consumeError(NameOrErr.takeError()); if (Name == "__nl_symbol_ptr") return populateIndirectSymbolPointersSection(cast<MachOObjectFile>(Obj), diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index f0de27ba14bb..523deb29b723 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -128,7 +128,10 @@ public: Error finalizeSection(const ObjectFile &Obj, unsigned SectionID, const SectionRef &Section) { StringRef Name; - Section.getName(Name); + if (Expected<StringRef> NameOrErr = Section.getName()) + Name = *NameOrErr; + else + consumeError(NameOrErr.takeError()); if (Name == "__jump_table") return populateJumpTable(cast<MachOObjectFile>(Obj), Section, SectionID); |