diff options
Diffstat (limited to 'lib/ExecutionEngine')
41 files changed, 2634 insertions, 5404 deletions
diff --git a/lib/ExecutionEngine/CMakeLists.txt b/lib/ExecutionEngine/CMakeLists.txt index 3102c7bd5822..dc2fe9bf97f3 100644 --- a/lib/ExecutionEngine/CMakeLists.txt +++ b/lib/ExecutionEngine/CMakeLists.txt @@ -3,12 +3,12 @@ add_llvm_library(LLVMExecutionEngine ExecutionEngine.cpp ExecutionEngineBindings.cpp + GDBRegistrationListener.cpp RTDyldMemoryManager.cpp TargetSelect.cpp ) add_subdirectory(Interpreter) -add_subdirectory(JIT) add_subdirectory(MCJIT) add_subdirectory(RuntimeDyld) diff --git a/lib/ExecutionEngine/EventListenerCommon.h b/lib/ExecutionEngine/EventListenerCommon.h index 66645d7297e9..6f17427052cd 100644 --- a/lib/ExecutionEngine/EventListenerCommon.h +++ b/lib/ExecutionEngine/EventListenerCommon.h @@ -1,67 +1,68 @@ -//===-- JIT.h - Abstract Execution Engine Interface -------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Common functionality for JITEventListener implementations -// -//===----------------------------------------------------------------------===// - -#ifndef EVENT_LISTENER_COMMON_H -#define EVENT_LISTENER_COMMON_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/Metadata.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/Support/Path.h" - -namespace llvm { - -namespace jitprofiling { - -class FilenameCache { - // Holds the filename of each Scope, so that we can pass a null-terminated - // string into oprofile. Use an AssertingVH rather than a ValueMap because we - // shouldn't be modifying any MDNodes while this map is alive. - DenseMap<AssertingVH<MDNode>, std::string> Filenames; - DenseMap<AssertingVH<MDNode>, std::string> Paths; - - public: - const char *getFilename(MDNode *Scope) { - std::string &Filename = Filenames[Scope]; - if (Filename.empty()) { - DIScope DIScope(Scope); - Filename = DIScope.getFilename(); - } - return Filename.c_str(); - } - - const char *getFullPath(MDNode *Scope) { - std::string &P = Paths[Scope]; - if (P.empty()) { - DIScope DIScope(Scope); - StringRef DirName = DIScope.getDirectory(); - StringRef FileName = DIScope.getFilename(); - SmallString<256> FullPath; - if (DirName != "." && DirName != "") { - FullPath = DirName; - } - if (FileName != "") { - sys::path::append(FullPath, FileName); - } - P = FullPath.str(); - } - return P.c_str(); - } -}; - -} // namespace jitprofiling - -} // namespace llvm - -#endif //EVENT_LISTENER_COMMON_H +//===-- JIT.h - Abstract Execution Engine Interface -------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Common functionality for JITEventListener implementations
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef EVENT_LISTENER_COMMON_H
+#define EVENT_LISTENER_COMMON_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Path.h"
+
+namespace llvm {
+
+namespace jitprofiling {
+
+class FilenameCache {
+ // Holds the filename of each Scope, so that we can pass a null-terminated
+ // string into oprofile.
+ DenseMap<const MDNode *, std::string> Filenames;
+ DenseMap<const MDNode *, std::string> Paths;
+
+ public:
+ const char *getFilename(MDNode *Scope) {
+ assert(Scope->isResolved() && "Expected Scope to be resolved");
+ std::string &Filename = Filenames[Scope];
+ if (Filename.empty()) {
+ DIScope DIScope(Scope);
+ Filename = DIScope.getFilename();
+ }
+ return Filename.c_str();
+ }
+
+ const char *getFullPath(MDNode *Scope) {
+ assert(Scope->isResolved() && "Expected Scope to be resolved");
+ std::string &P = Paths[Scope];
+ if (P.empty()) {
+ DIScope DIScope(Scope);
+ StringRef DirName = DIScope.getDirectory();
+ StringRef FileName = DIScope.getFilename();
+ SmallString<256> FullPath;
+ if (DirName != "." && DirName != "") {
+ FullPath = DirName;
+ }
+ if (FileName != "") {
+ sys::path::append(FullPath, FileName);
+ }
+ P = FullPath.str();
+ }
+ return P.c_str();
+ }
+};
+
+} // namespace jitprofiling
+
+} // namespace llvm
+
+#endif //EVENT_LISTENER_COMMON_H
diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp index b0e985d6dd54..32d20ea0e304 100644 --- a/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/lib/ExecutionEngine/ExecutionEngine.cpp @@ -16,14 +16,14 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Statistic.h" #include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Object/Archive.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" #include "llvm/Support/DynamicLibrary.h" @@ -42,27 +42,16 @@ using namespace llvm; STATISTIC(NumInitBytes, "Number of bytes of global vars initialized"); STATISTIC(NumGlobals , "Number of global vars initialized"); -// Pin the vtable to this file. -void ObjectCache::anchor() {} -void ObjectBuffer::anchor() {} -void ObjectBufferStream::anchor() {} - -ExecutionEngine *(*ExecutionEngine::JITCtor)( - Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM) = nullptr; ExecutionEngine *(*ExecutionEngine::MCJITCtor)( - Module *M, - std::string *ErrorStr, - RTDyldMemoryManager *MCJMM, - bool GVsWithCode, - TargetMachine *TM) = nullptr; -ExecutionEngine *(*ExecutionEngine::InterpCtor)(Module *M, + std::unique_ptr<Module> M, std::string *ErrorStr, + std::unique_ptr<RTDyldMemoryManager> MCJMM, + std::unique_ptr<TargetMachine> TM) = nullptr; +ExecutionEngine *(*ExecutionEngine::InterpCtor)(std::unique_ptr<Module> M, std::string *ErrorStr) =nullptr; -ExecutionEngine::ExecutionEngine(Module *M) +void JITEventListener::anchor() {} + +ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M) : EEState(*this), LazyFunctionCreator(nullptr) { CompilingLazily = false; @@ -77,14 +66,12 @@ ExecutionEngine::ExecutionEngine(Module *M) VerifyModules = false; #endif - Modules.push_back(M); assert(M && "Module is null?"); + Modules.push_back(std::move(M)); } ExecutionEngine::~ExecutionEngine() { clearAllGlobalMappings(); - for (unsigned i = 0, e = Modules.size(); i != e; ++i) - delete Modules[i]; } namespace { @@ -101,8 +88,8 @@ public: Type *ElTy = GV->getType()->getElementType(); size_t GVSize = (size_t)TD.getTypeAllocSize(ElTy); void *RawMemory = ::operator new( - DataLayout::RoundUpAlignment(sizeof(GVMemoryBlock), - TD.getPreferredAlignment(GV)) + RoundUpToAlignment(sizeof(GVMemoryBlock), + TD.getPreferredAlignment(GV)) + GVSize); new(RawMemory) GVMemoryBlock(GV); return static_cast<char*>(RawMemory) + sizeof(GVMemoryBlock); @@ -126,11 +113,20 @@ void ExecutionEngine::addObjectFile(std::unique_ptr<object::ObjectFile> O) { llvm_unreachable("ExecutionEngine subclass doesn't implement addObjectFile."); } +void +ExecutionEngine::addObjectFile(object::OwningBinary<object::ObjectFile> O) { + llvm_unreachable("ExecutionEngine subclass doesn't implement addObjectFile."); +} + +void ExecutionEngine::addArchive(object::OwningBinary<object::Archive> A) { + llvm_unreachable("ExecutionEngine subclass doesn't implement addArchive."); +} + bool ExecutionEngine::removeModule(Module *M) { - for(SmallVectorImpl<Module *>::iterator I = Modules.begin(), - E = Modules.end(); I != E; ++I) { - Module *Found = *I; + for (auto I = Modules.begin(), E = Modules.end(); I != E; ++I) { + Module *Found = I->get(); if (Found == M) { + I->release(); Modules.erase(I); clearGlobalMappingsFromModule(M); return true; @@ -254,19 +250,9 @@ const GlobalValue *ExecutionEngine::getGlobalValueAtAddress(void *Addr) { namespace { class ArgvArray { - char *Array; - std::vector<char*> Values; + std::unique_ptr<char[]> Array; + std::vector<std::unique_ptr<char[]>> Values; public: - ArgvArray() : Array(nullptr) {} - ~ArgvArray() { clear(); } - void clear() { - delete[] Array; - Array = nullptr; - for (size_t I = 0, E = Values.size(); I != E; ++I) { - delete[] Values[I]; - } - Values.clear(); - } /// Turn a vector of strings into a nice argv style array of pointers to null /// terminated strings. void *reset(LLVMContext &C, ExecutionEngine *EE, @@ -275,38 +261,39 @@ public: } // anonymous namespace void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE, const std::vector<std::string> &InputArgv) { - clear(); // Free the old contents. + Values.clear(); // Free the old contents. + Values.reserve(InputArgv.size()); unsigned PtrSize = EE->getDataLayout()->getPointerSize(); - Array = new char[(InputArgv.size()+1)*PtrSize]; + Array = make_unique<char[]>((InputArgv.size()+1)*PtrSize); - DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array << "\n"); + DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array.get() << "\n"); Type *SBytePtr = Type::getInt8PtrTy(C); for (unsigned i = 0; i != InputArgv.size(); ++i) { unsigned Size = InputArgv[i].size()+1; - char *Dest = new char[Size]; - Values.push_back(Dest); - DEBUG(dbgs() << "JIT: ARGV[" << i << "] = " << (void*)Dest << "\n"); + auto Dest = make_unique<char[]>(Size); + DEBUG(dbgs() << "JIT: ARGV[" << i << "] = " << (void*)Dest.get() << "\n"); - std::copy(InputArgv[i].begin(), InputArgv[i].end(), Dest); + std::copy(InputArgv[i].begin(), InputArgv[i].end(), Dest.get()); Dest[Size-1] = 0; // Endian safe: Array[i] = (PointerTy)Dest; - EE->StoreValueToMemory(PTOGV(Dest), (GenericValue*)(Array+i*PtrSize), - SBytePtr); + EE->StoreValueToMemory(PTOGV(Dest.get()), + (GenericValue*)(&Array[i*PtrSize]), SBytePtr); + Values.push_back(std::move(Dest)); } // Null terminate it EE->StoreValueToMemory(PTOGV(nullptr), - (GenericValue*)(Array+InputArgv.size()*PtrSize), + (GenericValue*)(&Array[InputArgv.size()*PtrSize]), SBytePtr); - return Array; + return Array.get(); } -void ExecutionEngine::runStaticConstructorsDestructors(Module *module, +void ExecutionEngine::runStaticConstructorsDestructors(Module &module, bool isDtors) { const char *Name = isDtors ? "llvm.global_dtors" : "llvm.global_ctors"; - GlobalVariable *GV = module->getNamedGlobal(Name); + GlobalVariable *GV = module.getNamedGlobal(Name); // If this global has internal linkage, or if it has a use, then it must be // an old-style (llvmgcc3) static ctor with __main linked in and in use. If @@ -344,8 +331,8 @@ void ExecutionEngine::runStaticConstructorsDestructors(Module *module, void ExecutionEngine::runStaticConstructorsDestructors(bool isDtors) { // Execute global ctors/dtors for each module in the program. - for (unsigned i = 0, e = Modules.size(); i != e; ++i) - runStaticConstructorsDestructors(Modules[i], isDtors); + for (std::unique_ptr<Module> &M : Modules) + runStaticConstructorsDestructors(*M, isDtors); } #ifndef NDEBUG @@ -406,55 +393,17 @@ int ExecutionEngine::runFunctionAsMain(Function *Fn, return runFunction(Fn, GVArgs).IntVal.getZExtValue(); } -ExecutionEngine *ExecutionEngine::create(Module *M, - bool ForceInterpreter, - std::string *ErrorStr, - CodeGenOpt::Level OptLevel, - bool GVsWithCode) { - - EngineBuilder EB = - EngineBuilder(M) - .setEngineKind(ForceInterpreter ? EngineKind::Interpreter - : EngineKind::Either) - .setErrorStr(ErrorStr) - .setOptLevel(OptLevel) - .setAllocateGVsWithCode(GVsWithCode); - - return EB.create(); +EngineBuilder::EngineBuilder(std::unique_ptr<Module> M) + : M(std::move(M)), MCJMM(nullptr) { + InitEngine(); } -/// createJIT - This is the factory method for creating a JIT for the current -/// machine, it does not fall back to the interpreter. This takes ownership -/// of the module. -ExecutionEngine *ExecutionEngine::createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - CodeGenOpt::Level OL, - bool GVsWithCode, - Reloc::Model RM, - CodeModel::Model CMM) { - if (!ExecutionEngine::JITCtor) { - if (ErrorStr) - *ErrorStr = "JIT has not been linked in."; - return nullptr; - } +EngineBuilder::~EngineBuilder() {} - // Use the defaults for extra parameters. Users can use EngineBuilder to - // set them. - EngineBuilder EB(M); - EB.setEngineKind(EngineKind::JIT); - EB.setErrorStr(ErrorStr); - EB.setRelocationModel(RM); - EB.setCodeModel(CMM); - EB.setAllocateGVsWithCode(GVsWithCode); - EB.setOptLevel(OL); - EB.setJITMemoryManager(JMM); - - // TODO: permit custom TargetOptions here - TargetMachine *TM = EB.selectTarget(); - if (!TM || (ErrorStr && ErrorStr->length() > 0)) return nullptr; - - return ExecutionEngine::JITCtor(M, ErrorStr, JMM, GVsWithCode, TM); +EngineBuilder &EngineBuilder::setMCJITMemoryManager( + std::unique_ptr<RTDyldMemoryManager> mcjmm) { + MCJMM = std::move(mcjmm); + return *this; } void EngineBuilder::InitEngine() { @@ -462,12 +411,9 @@ void EngineBuilder::InitEngine() { ErrorStr = nullptr; OptLevel = CodeGenOpt::Default; MCJMM = nullptr; - JMM = nullptr; Options = TargetOptions(); - AllocateGVsWithCode = false; RelocModel = Reloc::Default; CMModel = CodeModel::JITDefault; - UseMCJIT = false; // IR module verification is enabled by default in debug builds, and disabled // by default in release builds. @@ -485,13 +431,11 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { // to the function tells DynamicLibrary to load the program, not a library. if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, ErrorStr)) return nullptr; - - assert(!(JMM && MCJMM)); // If the user specified a memory manager but didn't specify which engine to // create, we assume they only want the JIT, and we fail if they only want // the interpreter. - if (JMM || MCJMM) { + if (MCJMM) { if (WhichEngine & EngineKind::JIT) WhichEngine = EngineKind::JIT; else { @@ -500,14 +444,6 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { return nullptr; } } - - if (MCJMM && ! UseMCJIT) { - if (ErrorStr) - *ErrorStr = - "Cannot create a legacy JIT with a runtime dyld memory " - "manager."; - return nullptr; - } // Unless the interpreter was explicitly selected or the JIT is not linked, // try making a JIT. @@ -520,13 +456,9 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { } ExecutionEngine *EE = nullptr; - if (UseMCJIT && ExecutionEngine::MCJITCtor) - EE = ExecutionEngine::MCJITCtor(M, ErrorStr, MCJMM ? MCJMM : JMM, - AllocateGVsWithCode, TheTM.release()); - else if (ExecutionEngine::JITCtor) - EE = ExecutionEngine::JITCtor(M, ErrorStr, JMM, - AllocateGVsWithCode, TheTM.release()); - + if (ExecutionEngine::MCJITCtor) + EE = ExecutionEngine::MCJITCtor(std::move(M), ErrorStr, std::move(MCJMM), + std::move(TheTM)); if (EE) { EE->setVerifyModules(VerifyModules); return EE; @@ -537,14 +469,13 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { // an interpreter instead. if (WhichEngine & EngineKind::Interpreter) { if (ExecutionEngine::InterpCtor) - return ExecutionEngine::InterpCtor(M, ErrorStr); + return ExecutionEngine::InterpCtor(std::move(M), ErrorStr); if (ErrorStr) *ErrorStr = "Interpreter has not been linked in."; return nullptr; } - if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::JITCtor && - !ExecutionEngine::MCJITCtor) { + if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::MCJITCtor) { if (ErrorStr) *ErrorStr = "JIT has not been linked in."; } @@ -890,9 +821,6 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { Result = PTOGV(getPointerToFunctionOrStub(const_cast<Function*>(F))); else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C)) Result = PTOGV(getOrEmitGlobalVariable(const_cast<GlobalVariable*>(GV))); - else if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) - Result = PTOGV(getPointerToBasicBlock(const_cast<BasicBlock*>( - BA->getBasicBlock()))); else llvm_unreachable("Unknown constant pointer type!"); break; diff --git a/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/lib/ExecutionEngine/ExecutionEngineBindings.cpp index 6ff1e7ac0634..aaa53f0c6953 100644 --- a/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -27,14 +27,6 @@ using namespace llvm; // Wrapping the C bindings types. DEFINE_SIMPLE_CONVERSION_FUNCTIONS(GenericValue, LLVMGenericValueRef) -inline TargetLibraryInfo *unwrap(LLVMTargetLibraryInfoRef P) { - return reinterpret_cast<TargetLibraryInfo*>(P); -} - -inline LLVMTargetLibraryInfoRef wrap(const TargetLibraryInfo *P) { - TargetLibraryInfo *X = const_cast<TargetLibraryInfo*>(P); - return reinterpret_cast<LLVMTargetLibraryInfoRef>(X); -} inline LLVMTargetMachineRef wrap(const TargetMachine *P) { return @@ -110,7 +102,7 @@ LLVMBool LLVMCreateExecutionEngineForModule(LLVMExecutionEngineRef *OutEE, LLVMModuleRef M, char **OutError) { std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::Either) .setErrorStr(&Error); if (ExecutionEngine *EE = builder.create()){ @@ -125,7 +117,7 @@ LLVMBool LLVMCreateInterpreterForModule(LLVMExecutionEngineRef *OutInterp, LLVMModuleRef M, char **OutError) { std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::Interpreter) .setErrorStr(&Error); if (ExecutionEngine *Interp = builder.create()) { @@ -141,7 +133,7 @@ LLVMBool LLVMCreateJITCompilerForModule(LLVMExecutionEngineRef *OutJIT, unsigned OptLevel, char **OutError) { std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::JIT) .setErrorStr(&Error) .setOptLevel((CodeGenOpt::Level)OptLevel); @@ -189,15 +181,15 @@ LLVMBool LLVMCreateMCJITCompilerForModule( targetOptions.EnableFastISel = options.EnableFastISel; std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::JIT) .setErrorStr(&Error) - .setUseMCJIT(true) .setOptLevel((CodeGenOpt::Level)options.OptLevel) .setCodeModel(unwrap(options.CodeModel)) .setTargetOptions(targetOptions); if (options.MCJMM) - builder.setMCJITMemoryManager(unwrap(options.MCJMM)); + builder.setMCJITMemoryManager( + std::unique_ptr<RTDyldMemoryManager>(unwrap(options.MCJMM))); if (ExecutionEngine *JIT = builder.create()) { *OutJIT = wrap(JIT); return 0; @@ -275,11 +267,10 @@ LLVMGenericValueRef LLVMRunFunction(LLVMExecutionEngineRef EE, LLVMValueRef F, } void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE, LLVMValueRef F) { - unwrap(EE)->freeMachineCodeForFunction(unwrap<Function>(F)); } void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M){ - unwrap(EE)->addModule(unwrap(M)); + unwrap(EE)->addModule(std::unique_ptr<Module>(unwrap(M))); } void LLVMAddModuleProvider(LLVMExecutionEngineRef EE, LLVMModuleProviderRef MP){ @@ -314,7 +305,7 @@ LLVMBool LLVMFindFunction(LLVMExecutionEngineRef EE, const char *Name, void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE, LLVMValueRef Fn) { - return unwrap(EE)->recompileAndRelinkFunction(unwrap<Function>(Fn)); + return nullptr; } LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) { @@ -337,6 +328,14 @@ void *LLVMGetPointerToGlobal(LLVMExecutionEngineRef EE, LLVMValueRef Global) { return unwrap(EE)->getPointerToGlobal(unwrap<GlobalValue>(Global)); } +uint64_t LLVMGetGlobalValueAddress(LLVMExecutionEngineRef EE, const char *Name) { + return unwrap(EE)->getGlobalValueAddress(Name); +} + +uint64_t LLVMGetFunctionAddress(LLVMExecutionEngineRef EE, const char *Name) { + return unwrap(EE)->getFunctionAddress(Name); +} + /*===-- Operations on memory managers -------------------------------------===*/ namespace { diff --git a/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp b/lib/ExecutionEngine/GDBRegistrationListener.cpp index 85465715873f..8ef878c2878e 100644 --- a/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp +++ b/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -1,4 +1,4 @@ -//===-- GDBRegistrar.cpp - Registers objects with GDB ---------------------===// +//===----- GDBRegistrationListener.cpp - Registers objects with GDB -------===// // // The LLVM Compiler Infrastructure // @@ -7,15 +7,17 @@ // //===----------------------------------------------------------------------===// -#include "JITRegistrar.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/Object/ObjectFile.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/MutexGuard.h" -#include "llvm/Support/ManagedStatic.h" using namespace llvm; +using namespace llvm::object; // This must be kept in sync with gdb/gdb/jit.h . extern "C" { @@ -60,37 +62,59 @@ extern "C" { namespace { +struct RegisteredObjectInfo { + RegisteredObjectInfo() {} + + RegisteredObjectInfo(std::size_t Size, jit_code_entry *Entry, + OwningBinary<ObjectFile> Obj) + : Size(Size), Entry(Entry), Obj(std::move(Obj)) {} + + RegisteredObjectInfo(RegisteredObjectInfo &&Other) + : Size(Other.Size), Entry(Other.Entry), Obj(std::move(Other.Obj)) {} + + RegisteredObjectInfo& operator=(RegisteredObjectInfo &&Other) { + Size = Other.Size; + Entry = Other.Entry; + Obj = std::move(Other.Obj); + return *this; + } + + std::size_t Size; + jit_code_entry *Entry; + OwningBinary<ObjectFile> Obj; +}; + // Buffer for an in-memory object file in executable memory -typedef llvm::DenseMap< const char*, - std::pair<std::size_t, jit_code_entry*> > +typedef llvm::DenseMap< const char*, RegisteredObjectInfo> RegisteredObjectBufferMap; /// Global access point for the JIT debugging interface designed for use with a /// singleton toolbox. Handles thread-safe registration and deregistration of /// object files that are in executable memory managed by the client of this /// class. -class GDBJITRegistrar : public JITRegistrar { +class GDBJITRegistrationListener : public JITEventListener { /// A map of in-memory object files that have been registered with the /// JIT interface. RegisteredObjectBufferMap ObjectBufferMap; public: /// Instantiates the JIT service. - GDBJITRegistrar() : ObjectBufferMap() {} + GDBJITRegistrationListener() : ObjectBufferMap() {} /// Unregisters each object that was previously registered and releases all /// internal resources. - virtual ~GDBJITRegistrar(); + virtual ~GDBJITRegistrationListener(); /// Creates an entry in the JIT registry for the buffer @p Object, /// which must contain an object file in executable memory with any /// debug information for the debugger. - void registerObject(const ObjectBuffer &Object) override; + void NotifyObjectEmitted(const ObjectFile &Object, + const RuntimeDyld::LoadedObjectInfo &L) override; /// Removes the internal registration of @p Object, and /// frees associated resources. /// Returns true if @p Object was found in ObjectBufferMap. - bool deregisterObject(const ObjectBuffer &Object) override; + void NotifyFreeingObject(const ObjectFile &Object) override; private: /// Deregister the debug info for the given object file from the debugger @@ -101,7 +125,7 @@ private: /// Lock used to serialize all jit registration events, since they /// modify global variables. -llvm::sys::Mutex JITDebugLock; +ManagedStatic<sys::Mutex> JITDebugLock; /// Do the registration. void NotifyDebugger(jit_code_entry* JITCodeEntry) { @@ -119,10 +143,11 @@ void NotifyDebugger(jit_code_entry* JITCodeEntry) { __jit_debug_register_code(); } -GDBJITRegistrar::~GDBJITRegistrar() { +GDBJITRegistrationListener::~GDBJITRegistrationListener() { // Free all registered object files. - llvm::MutexGuard locked(JITDebugLock); - for (RegisteredObjectBufferMap::iterator I = ObjectBufferMap.begin(), E = ObjectBufferMap.end(); + llvm::MutexGuard locked(*JITDebugLock); + for (RegisteredObjectBufferMap::iterator I = ObjectBufferMap.begin(), + E = ObjectBufferMap.end(); I != E; ++I) { // Call the private method that doesn't update the map so our iterator // doesn't break. @@ -131,14 +156,24 @@ GDBJITRegistrar::~GDBJITRegistrar() { ObjectBufferMap.clear(); } -void GDBJITRegistrar::registerObject(const ObjectBuffer &Object) { +void GDBJITRegistrationListener::NotifyObjectEmitted( + const ObjectFile &Object, + const RuntimeDyld::LoadedObjectInfo &L) { + + OwningBinary<ObjectFile> DebugObj = L.getObjectForDebug(Object); + + // Bail out if debug objects aren't supported. + if (!DebugObj.getBinary()) + return; + + const char *Buffer = DebugObj.getBinary()->getMemoryBufferRef().getBufferStart(); + size_t Size = DebugObj.getBinary()->getMemoryBufferRef().getBufferSize(); - const char *Buffer = Object.getBufferStart(); - size_t Size = Object.getBufferSize(); + const char *Key = Object.getMemoryBufferRef().getBufferStart(); - assert(Buffer && "Attempt to register a null object with a debugger."); - llvm::MutexGuard locked(JITDebugLock); - assert(ObjectBufferMap.find(Buffer) == ObjectBufferMap.end() && + assert(Key && "Attempt to register a null object with a debugger."); + llvm::MutexGuard locked(*JITDebugLock); + assert(ObjectBufferMap.find(Key) == ObjectBufferMap.end() && "Second attempt to perform debug registration."); jit_code_entry* JITCodeEntry = new jit_code_entry(); @@ -149,28 +184,27 @@ void GDBJITRegistrar::registerObject(const ObjectBuffer &Object) { JITCodeEntry->symfile_addr = Buffer; JITCodeEntry->symfile_size = Size; - ObjectBufferMap[Buffer] = std::make_pair(Size, JITCodeEntry); + ObjectBufferMap[Key] = RegisteredObjectInfo(Size, JITCodeEntry, + std::move(DebugObj)); NotifyDebugger(JITCodeEntry); } } -bool GDBJITRegistrar::deregisterObject(const ObjectBuffer& Object) { - const char *Buffer = Object.getBufferStart(); - llvm::MutexGuard locked(JITDebugLock); - RegisteredObjectBufferMap::iterator I = ObjectBufferMap.find(Buffer); +void GDBJITRegistrationListener::NotifyFreeingObject(const ObjectFile& Object) { + const char *Key = Object.getMemoryBufferRef().getBufferStart(); + llvm::MutexGuard locked(*JITDebugLock); + RegisteredObjectBufferMap::iterator I = ObjectBufferMap.find(Key); if (I != ObjectBufferMap.end()) { deregisterObjectInternal(I); ObjectBufferMap.erase(I); - return true; } - return false; } -void GDBJITRegistrar::deregisterObjectInternal( +void GDBJITRegistrationListener::deregisterObjectInternal( RegisteredObjectBufferMap::iterator I) { - jit_code_entry*& JITCodeEntry = I->second.second; + jit_code_entry*& JITCodeEntry = I->second.Entry; // Do the unregistration. { @@ -200,14 +234,14 @@ void GDBJITRegistrar::deregisterObjectInternal( JITCodeEntry = nullptr; } -llvm::ManagedStatic<GDBJITRegistrar> TheRegistrar; +llvm::ManagedStatic<GDBJITRegistrationListener> GDBRegListener; } // end namespace namespace llvm { -JITRegistrar& JITRegistrar::getGDBRegistrar() { - return *TheRegistrar; +JITEventListener* JITEventListener::createGDBRegistrationListener() { + return &*GDBRegListener; } } // namespace llvm diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp index 4e22a8b3ea0d..ee9096b31e95 100644 --- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp +++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp @@ -13,25 +13,24 @@ //===----------------------------------------------------------------------===// #include "llvm/Config/config.h" +#include "EventListenerCommon.h" +#include "IntelJITEventsWrapper.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/DebugInfo/DIContext.h" #include "llvm/ExecutionEngine/JITEventListener.h" - #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/Metadata.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/DebugInfo/DIContext.h" -#include "llvm/ExecutionEngine/ObjectImage.h" +#include "llvm/IR/ValueHandle.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/Errno.h" -#include "llvm/IR/ValueHandle.h" -#include "EventListenerCommon.h" -#include "IntelJITEventsWrapper.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; using namespace llvm::jitprofiling; +using namespace llvm::object; #define DEBUG_TYPE "amplifier-jit-event-listener" @@ -48,6 +47,7 @@ class IntelJITEventListener : public JITEventListener { typedef DenseMap<const void *, MethodAddressVector> ObjectMap; ObjectMap LoadedObjectMap; + std::map<const char*, OwningBinary<ObjectFile>> DebugObjects; public: IntelJITEventListener(IntelJITEventsWrapper* libraryWrapper) { @@ -57,29 +57,12 @@ public: ~IntelJITEventListener() { } - virtual void NotifyFunctionEmitted(const Function &F, - void *FnStart, size_t FnSize, - const EmittedFunctionDetails &Details); + void NotifyObjectEmitted(const ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &L) override; - virtual void NotifyFreeingMachineCode(void *OldPtr); - - virtual void NotifyObjectEmitted(const ObjectImage &Obj); - - virtual void NotifyFreeingObject(const ObjectImage &Obj); + void NotifyFreeingObject(const ObjectFile &Obj) override; }; -static LineNumberInfo LineStartToIntelJITFormat( - uintptr_t StartAddress, - uintptr_t Address, - DebugLoc Loc) { - LineNumberInfo Result; - - Result.Offset = Address - StartAddress; - Result.LineNumber = Loc.getLine(); - - return Result; -} - static LineNumberInfo DILineInfoToIntelJITFormat(uintptr_t StartAddress, uintptr_t Address, DILineInfo Line) { @@ -113,97 +96,29 @@ static iJIT_Method_Load FunctionDescToIntelJITFormat( return Result; } -// Adds the just-emitted function to the symbol table. -void IntelJITEventListener::NotifyFunctionEmitted( - const Function &F, void *FnStart, size_t FnSize, - const EmittedFunctionDetails &Details) { - iJIT_Method_Load FunctionMessage = FunctionDescToIntelJITFormat(*Wrapper, - F.getName().data(), - reinterpret_cast<uint64_t>(FnStart), - FnSize); - - std::vector<LineNumberInfo> LineInfo; - - if (!Details.LineStarts.empty()) { - // Now convert the line number information from the address/DebugLoc - // format in Details to the offset/lineno in Intel JIT API format. - - LineInfo.reserve(Details.LineStarts.size() + 1); - - DebugLoc FirstLoc = Details.LineStarts[0].Loc; - assert(!FirstLoc.isUnknown() - && "LineStarts should not contain unknown DebugLocs"); - - MDNode *FirstLocScope = FirstLoc.getScope(F.getContext()); - DISubprogram FunctionDI = getDISubprogram(FirstLocScope); - if (FunctionDI.Verify()) { - FunctionMessage.source_file_name = const_cast<char*>( - Filenames.getFullPath(FirstLocScope)); - - LineNumberInfo FirstLine; - FirstLine.Offset = 0; - FirstLine.LineNumber = FunctionDI.getLineNumber(); - LineInfo.push_back(FirstLine); - } - - for (std::vector<EmittedFunctionDetails::LineStart>::const_iterator I = - Details.LineStarts.begin(), E = Details.LineStarts.end(); - I != E; ++I) { - // This implementation ignores the DebugLoc filename because the Intel - // JIT API does not support multiple source files associated with a single - // JIT function - LineInfo.push_back(LineStartToIntelJITFormat( - reinterpret_cast<uintptr_t>(FnStart), - I->Address, - I->Loc)); - - // If we have no file name yet for the function, use the filename from - // the first instruction that has one - if (FunctionMessage.source_file_name == 0) { - MDNode *scope = I->Loc.getScope( - Details.MF->getFunction()->getContext()); - FunctionMessage.source_file_name = const_cast<char*>( - Filenames.getFullPath(scope)); - } - } - - FunctionMessage.line_number_size = LineInfo.size(); - FunctionMessage.line_number_table = &*LineInfo.begin(); - } else { - FunctionMessage.line_number_size = 0; - FunctionMessage.line_number_table = 0; - } +void IntelJITEventListener::NotifyObjectEmitted( + const ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &L) { - Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, - &FunctionMessage); - MethodIDs[FnStart] = FunctionMessage.method_id; -} + OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj); + const ObjectFile &DebugObj = *DebugObjOwner.getBinary(); -void IntelJITEventListener::NotifyFreeingMachineCode(void *FnStart) { - MethodIDMap::iterator I = MethodIDs.find(FnStart); - if (I != MethodIDs.end()) { - Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_UNLOAD_START, &I->second); - MethodIDs.erase(I); - } -} - -void IntelJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { // Get the address of the object image for use as a unique identifier - const void* ObjData = Obj.getData().data(); - DIContext* Context = DIContext::getDWARFContext(Obj.getObjectFile()); + const void* ObjData = DebugObj.getData().data(); + DIContext* Context = DIContext::getDWARFContext(DebugObj); MethodAddressVector Functions; // Use symbol info to iterate functions in the object. - for (object::symbol_iterator I = Obj.begin_symbols(), - E = Obj.end_symbols(); + for (symbol_iterator I = DebugObj.symbol_begin(), + E = DebugObj.symbol_end(); I != E; ++I) { std::vector<LineNumberInfo> LineInfo; std::string SourceFileName; - object::SymbolRef::Type SymType; + SymbolRef::Type SymType; if (I->getType(SymType)) continue; - if (SymType == object::SymbolRef::ST_Function) { + if (SymType == SymbolRef::ST_Function) { StringRef Name; uint64_t Addr; uint64_t Size; @@ -233,6 +148,18 @@ void IntelJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { FunctionMessage.line_number_size = 0; FunctionMessage.line_number_table = 0; } else { + // Source line information for the address range is provided as + // a code offset for the start of the corresponding sub-range and + // a source line. JIT API treats offsets in LineNumberInfo structures + // as the end of the corresponding code region. The start of the code + // is taken from the previous element. Need to shift the elements. + + LineNumberInfo last = LineInfo.back(); + last.Offset = FunctionMessage.method_size; + LineInfo.push_back(last); + for (size_t i = LineInfo.size() - 2; i > 0; --i) + LineInfo[i].LineNumber = LineInfo[i - 1].LineNumber; + SourceFileName = Lines.front().second.FileName; FunctionMessage.source_file_name = const_cast<char *>(SourceFileName.c_str()); FunctionMessage.line_number_size = LineInfo.size(); @@ -254,11 +181,18 @@ void IntelJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { // registered function addresses for each loaded object. We will // use the MethodIDs map to get the registered ID for each function. LoadedObjectMap[ObjData] = Functions; + DebugObjects[Obj.getData().data()] = std::move(DebugObjOwner); } -void IntelJITEventListener::NotifyFreeingObject(const ObjectImage &Obj) { +void IntelJITEventListener::NotifyFreeingObject(const ObjectFile &Obj) { + // This object may not have been registered with the listener. If it wasn't, + // bail out. + if (DebugObjects.find(Obj.getData().data()) == DebugObjects.end()) + return; + // Get the address of the object image for use as a unique identifier - const void* ObjData = Obj.getData().data(); + const ObjectFile &DebugObj = *DebugObjects[Obj.getData().data()].getBinary(); + const void* ObjData = DebugObj.getData().data(); // Get the object's function list from LoadedObjectMap ObjectMap::iterator OI = LoadedObjectMap.find(ObjData); @@ -282,6 +216,7 @@ void IntelJITEventListener::NotifyFreeingObject(const ObjectImage &Obj) { // Erase the object from LoadedObjectMap LoadedObjectMap.erase(OI); + DebugObjects.erase(Obj.getData().data()); } } // anonymous namespace. diff --git a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt index 9c06fdae8689..e36493ebbaf6 100644 --- a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt +++ b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt @@ -21,3 +21,4 @@ type = OptionalLibrary name = IntelJITEvents parent = ExecutionEngine +required_libraries = Core DebugInfo Support diff --git a/lib/ExecutionEngine/Interpreter/CMakeLists.txt b/lib/ExecutionEngine/Interpreter/CMakeLists.txt index 74df8f0f37a5..1aac3ac7fdc9 100644 --- a/lib/ExecutionEngine/Interpreter/CMakeLists.txt +++ b/lib/ExecutionEngine/Interpreter/CMakeLists.txt @@ -13,7 +13,7 @@ add_llvm_library(LLVMInterpreter ) if( LLVM_ENABLE_FFI ) - target_link_libraries( LLVMInterpreter ${FFI_LIBRARY_PATH} ) + target_link_libraries( LLVMInterpreter ${cmake_2_8_12_PRIVATE} ${FFI_LIBRARY_PATH} ) endif() add_dependencies(LLVMInterpreter intrinsics_gen) diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index 671bbeea8d0d..b0221019b559 100644 --- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -28,6 +28,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Mutex.h" +#include "llvm/Support/UniqueLock.h" #include <cmath> #include <csignal> #include <cstdio> @@ -51,7 +52,7 @@ static ManagedStatic<sys::Mutex> FunctionsLock; typedef GenericValue (*ExFunc)(FunctionType *, const std::vector<GenericValue> &); static ManagedStatic<std::map<const Function *, ExFunc> > ExportedFunctions; -static std::map<std::string, ExFunc> FuncNames; +static ManagedStatic<std::map<std::string, ExFunc> > FuncNames; #ifdef USE_LIBFFI typedef void (*RawFunc)(); @@ -97,9 +98,9 @@ static ExFunc lookupFunction(const Function *F) { ExtName += "_" + F->getName().str(); sys::ScopedLock Writer(*FunctionsLock); - ExFunc FnPtr = FuncNames[ExtName]; + ExFunc FnPtr = (*FuncNames)[ExtName]; if (!FnPtr) - FnPtr = FuncNames["lle_X_" + F->getName().str()]; + FnPtr = (*FuncNames)["lle_X_" + F->getName().str()]; if (!FnPtr) // Try calling a generic function... if it exists... FnPtr = (ExFunc)(intptr_t) sys::DynamicLibrary::SearchForAddressOfSymbol("lle_X_" + @@ -248,14 +249,14 @@ GenericValue Interpreter::callExternalFunction(Function *F, const std::vector<GenericValue> &ArgVals) { TheInterpreter = this; - FunctionsLock->acquire(); + unique_lock<sys::Mutex> Guard(*FunctionsLock); // Do a lookup to see if the function is in our cache... this should just be a // deferred annotation! std::map<const Function *, ExFunc>::iterator FI = ExportedFunctions->find(F); if (ExFunc Fn = (FI == ExportedFunctions->end()) ? lookupFunction(F) : FI->second) { - FunctionsLock->release(); + Guard.unlock(); return Fn(F->getFunctionType(), ArgVals); } @@ -273,7 +274,7 @@ GenericValue Interpreter::callExternalFunction(Function *F, RawFn = RF->second; } - FunctionsLock->release(); + Guard.unlock(); GenericValue Result; if (RawFn != 0 && ffiInvoke(RawFn, F, ArgVals, getDataLayout(), Result)) @@ -497,15 +498,15 @@ static GenericValue lle_X_memcpy(FunctionType *FT, void Interpreter::initializeExternalFunctions() { sys::ScopedLock Writer(*FunctionsLock); - FuncNames["lle_X_atexit"] = lle_X_atexit; - FuncNames["lle_X_exit"] = lle_X_exit; - FuncNames["lle_X_abort"] = lle_X_abort; - - FuncNames["lle_X_printf"] = lle_X_printf; - FuncNames["lle_X_sprintf"] = lle_X_sprintf; - FuncNames["lle_X_sscanf"] = lle_X_sscanf; - FuncNames["lle_X_scanf"] = lle_X_scanf; - FuncNames["lle_X_fprintf"] = lle_X_fprintf; - FuncNames["lle_X_memset"] = lle_X_memset; - FuncNames["lle_X_memcpy"] = lle_X_memcpy; + (*FuncNames)["lle_X_atexit"] = lle_X_atexit; + (*FuncNames)["lle_X_exit"] = lle_X_exit; + (*FuncNames)["lle_X_abort"] = lle_X_abort; + + (*FuncNames)["lle_X_printf"] = lle_X_printf; + (*FuncNames)["lle_X_sprintf"] = lle_X_sprintf; + (*FuncNames)["lle_X_sscanf"] = lle_X_sscanf; + (*FuncNames)["lle_X_scanf"] = lle_X_scanf; + (*FuncNames)["lle_X_fprintf"] = lle_X_fprintf; + (*FuncNames)["lle_X_memset"] = lle_X_memset; + (*FuncNames)["lle_X_memcpy"] = lle_X_memcpy; } diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/lib/ExecutionEngine/Interpreter/Interpreter.cpp index 814efcc27fcb..8562981b629a 100644 --- a/lib/ExecutionEngine/Interpreter/Interpreter.cpp +++ b/lib/ExecutionEngine/Interpreter/Interpreter.cpp @@ -30,9 +30,10 @@ static struct RegisterInterp { extern "C" void LLVMLinkInInterpreter() { } -/// create - Create a new interpreter object. This can never fail. +/// Create a new interpreter object. /// -ExecutionEngine *Interpreter::create(Module *M, std::string* ErrStr) { +ExecutionEngine *Interpreter::create(std::unique_ptr<Module> M, + std::string *ErrStr) { // Tell this Module to materialize everything and release the GVMaterializer. if (std::error_code EC = M->materializeAllPermanently()) { if (ErrStr) @@ -41,15 +42,15 @@ ExecutionEngine *Interpreter::create(Module *M, std::string* ErrStr) { return nullptr; } - return new Interpreter(M); + return new Interpreter(std::move(M)); } //===----------------------------------------------------------------------===// // Interpreter ctor - Initialize stuff // -Interpreter::Interpreter(Module *M) - : ExecutionEngine(M), TD(M) { - +Interpreter::Interpreter(std::unique_ptr<Module> M) + : ExecutionEngine(std::move(M)), TD(Modules.back().get()) { + memset(&ExitValue.Untyped, 0, sizeof(ExitValue.Untyped)); setDataLayout(&TD); // Initialize the "backend" diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.h b/lib/ExecutionEngine/Interpreter/Interpreter.h index 2145cde05fbf..2be9c5979d80 100644 --- a/lib/ExecutionEngine/Interpreter/Interpreter.h +++ b/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLI_INTERPRETER_H -#define LLI_INTERPRETER_H +#ifndef LLVM_LIB_EXECUTIONENGINE_INTERPRETER_INTERPRETER_H +#define LLVM_LIB_EXECUTIONENGINE_INTERPRETER_INTERPRETER_H #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/ExecutionEngine/GenericValue.h" @@ -37,29 +37,24 @@ typedef generic_gep_type_iterator<User::const_op_iterator> gep_type_iterator; // stack, which causes the dtor to be run, which frees all the alloca'd memory. // class AllocaHolder { - friend class AllocaHolderHandle; - std::vector<void*> Allocations; - unsigned RefCnt; + std::vector<void *> Allocations; + public: - AllocaHolder() : RefCnt(0) {} - void add(void *mem) { Allocations.push_back(mem); } - ~AllocaHolder() { - for (unsigned i = 0; i < Allocations.size(); ++i) - free(Allocations[i]); + AllocaHolder() {} + + // Make this type move-only. Define explicit move special members for MSVC. + AllocaHolder(AllocaHolder &&RHS) : Allocations(std::move(RHS.Allocations)) {} + AllocaHolder &operator=(AllocaHolder &&RHS) { + Allocations = std::move(RHS.Allocations); + return *this; } -}; -// AllocaHolderHandle gives AllocaHolder value semantics so we can stick it into -// a vector... -// -class AllocaHolderHandle { - AllocaHolder *H; -public: - AllocaHolderHandle() : H(new AllocaHolder()) { H->RefCnt++; } - AllocaHolderHandle(const AllocaHolderHandle &AH) : H(AH.H) { H->RefCnt++; } - ~AllocaHolderHandle() { if (--H->RefCnt == 0) delete H; } + ~AllocaHolder() { + for (void *Allocation : Allocations) + free(Allocation); + } - void add(void *mem) { H->add(mem); } + void add(void *Mem) { Allocations.push_back(Mem); } }; typedef std::vector<GenericValue> ValuePlaneTy; @@ -71,11 +66,29 @@ struct ExecutionContext { Function *CurFunction;// The currently executing function BasicBlock *CurBB; // The currently executing BB BasicBlock::iterator CurInst; // The next instruction to execute - std::map<Value *, GenericValue> Values; // LLVM values used in this invocation - std::vector<GenericValue> VarArgs; // Values passed through an ellipsis CallSite Caller; // Holds the call that called subframes. // NULL if main func or debugger invoked fn - AllocaHolderHandle Allocas; // Track memory allocated by alloca + std::map<Value *, GenericValue> Values; // LLVM values used in this invocation + std::vector<GenericValue> VarArgs; // Values passed through an ellipsis + AllocaHolder Allocas; // Track memory allocated by alloca + + ExecutionContext() : CurFunction(nullptr), CurBB(nullptr), CurInst(nullptr) {} + + ExecutionContext(ExecutionContext &&O) + : CurFunction(O.CurFunction), CurBB(O.CurBB), CurInst(O.CurInst), + Caller(O.Caller), Values(std::move(O.Values)), + VarArgs(std::move(O.VarArgs)), Allocas(std::move(O.Allocas)) {} + + ExecutionContext &operator=(ExecutionContext &&O) { + CurFunction = O.CurFunction; + CurBB = O.CurBB; + CurInst = O.CurInst; + Caller = O.Caller; + Values = std::move(O.Values); + VarArgs = std::move(O.VarArgs); + Allocas = std::move(O.Allocas); + return *this; + } }; // Interpreter - This class represents the entirety of the interpreter. @@ -94,7 +107,7 @@ class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> { std::vector<Function*> AtExitHandlers; public: - explicit Interpreter(Module *M); + explicit Interpreter(std::unique_ptr<Module> M); ~Interpreter(); /// runAtExitHandlers - Run any functions registered by the program's calls to @@ -105,33 +118,23 @@ public: static void Register() { InterpCtor = create; } - - /// create - Create an interpreter ExecutionEngine. This can never fail. + + /// Create an interpreter ExecutionEngine. /// - static ExecutionEngine *create(Module *M, std::string *ErrorStr = nullptr); + static ExecutionEngine *create(std::unique_ptr<Module> M, + std::string *ErrorStr = nullptr); /// run - Start execution with the specified function and arguments. /// GenericValue runFunction(Function *F, const std::vector<GenericValue> &ArgValues) override; - void *getPointerToNamedFunction(const std::string &Name, + void *getPointerToNamedFunction(StringRef Name, bool AbortOnFailure = true) override { // FIXME: not implemented. return nullptr; } - /// recompileAndRelinkFunction - For the interpreter, functions are always - /// up-to-date. - /// - void *recompileAndRelinkFunction(Function *F) override { - return getPointerToFunction(F); - } - - /// freeMachineCodeForFunction - The interpreter does not generate any code. - /// - void freeMachineCodeForFunction(Function *F) override { } - // Methods used to execute code: // Place a call on the stack void callFunction(Function *F, const std::vector<GenericValue> &ArgVals); @@ -213,7 +216,6 @@ private: // Helper functions void SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF); void *getPointerToFunction(Function *F) override { return (void*)F; } - void *getPointerToBasicBlock(BasicBlock *BB) override { return (void*)BB; } void initializeExecutionEngine() { } void initializeExternalFunctions(); diff --git a/lib/ExecutionEngine/JIT/CMakeLists.txt b/lib/ExecutionEngine/JIT/CMakeLists.txt deleted file mode 100644 index e16baede50fd..000000000000 --- a/lib/ExecutionEngine/JIT/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# TODO: Support other architectures. See Makefile. -add_definitions(-DENABLE_X86_JIT) - -add_llvm_library(LLVMJIT - JIT.cpp - JITEmitter.cpp - JITMemoryManager.cpp - ) diff --git a/lib/ExecutionEngine/JIT/JIT.cpp b/lib/ExecutionEngine/JIT/JIT.cpp deleted file mode 100644 index 83ec9784b98e..000000000000 --- a/lib/ExecutionEngine/JIT/JIT.cpp +++ /dev/null @@ -1,695 +0,0 @@ -//===-- JIT.cpp - LLVM Just in Time Compiler ------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This tool implements a just-in-time compiler for LLVM, allowing direct -// execution of LLVM bitcode in an efficient manner. -// -//===----------------------------------------------------------------------===// - -#include "JIT.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/Config/config.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/MutexGuard.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" - -using namespace llvm; - -#ifdef __APPLE__ -// Apple gcc defaults to -fuse-cxa-atexit (i.e. calls __cxa_atexit instead -// of atexit). It passes the address of linker generated symbol __dso_handle -// to the function. -// This configuration change happened at version 5330. -# include <AvailabilityMacros.h> -# if defined(MAC_OS_X_VERSION_10_4) && \ - ((MAC_OS_X_VERSION_MIN_REQUIRED > MAC_OS_X_VERSION_10_4) || \ - (MAC_OS_X_VERSION_MIN_REQUIRED == MAC_OS_X_VERSION_10_4 && \ - __APPLE_CC__ >= 5330)) -# ifndef HAVE___DSO_HANDLE -# define HAVE___DSO_HANDLE 1 -# endif -# endif -#endif - -#if HAVE___DSO_HANDLE -extern void *__dso_handle __attribute__ ((__visibility__ ("hidden"))); -#endif - -namespace { - -static struct RegisterJIT { - RegisterJIT() { JIT::Register(); } -} JITRegistrator; - -} - -extern "C" void LLVMLinkInJIT() { -} - -/// createJIT - This is the factory method for creating a JIT for the current -/// machine, it does not fall back to the interpreter. This takes ownership -/// of the module. -ExecutionEngine *JIT::createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM) { - // Try to register the program as a source of symbols to resolve against. - // - // FIXME: Don't do this here. - sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr); - - // If the target supports JIT code generation, create the JIT. - if (TargetJITInfo *TJ = TM->getJITInfo()) { - return new JIT(M, *TM, *TJ, JMM, GVsWithCode); - } else { - if (ErrorStr) - *ErrorStr = "target does not support JIT code generation"; - return nullptr; - } -} - -namespace { -/// This class supports the global getPointerToNamedFunction(), which allows -/// bugpoint or gdb users to search for a function by name without any context. -class JitPool { - SmallPtrSet<JIT*, 1> JITs; // Optimize for process containing just 1 JIT. - mutable sys::Mutex Lock; -public: - void Add(JIT *jit) { - MutexGuard guard(Lock); - JITs.insert(jit); - } - void Remove(JIT *jit) { - MutexGuard guard(Lock); - JITs.erase(jit); - } - void *getPointerToNamedFunction(const char *Name) const { - MutexGuard guard(Lock); - assert(JITs.size() != 0 && "No Jit registered"); - //search function in every instance of JIT - for (SmallPtrSet<JIT*, 1>::const_iterator Jit = JITs.begin(), - end = JITs.end(); - Jit != end; ++Jit) { - if (Function *F = (*Jit)->FindFunctionNamed(Name)) - return (*Jit)->getPointerToFunction(F); - } - // The function is not available : fallback on the first created (will - // search in symbol of the current program/library) - return (*JITs.begin())->getPointerToNamedFunction(Name); - } -}; -ManagedStatic<JitPool> AllJits; -} -extern "C" { - // getPointerToNamedFunction - This function is used as a global wrapper to - // JIT::getPointerToNamedFunction for the purpose of resolving symbols when - // bugpoint is debugging the JIT. In that scenario, we are loading an .so and - // need to resolve function(s) that are being mis-codegenerated, so we need to - // resolve their addresses at runtime, and this is the way to do it. - void *getPointerToNamedFunction(const char *Name) { - return AllJits->getPointerToNamedFunction(Name); - } -} - -JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji, - JITMemoryManager *jmm, bool GVsWithCode) - : ExecutionEngine(M), TM(tm), TJI(tji), - JMM(jmm ? jmm : JITMemoryManager::CreateDefaultMemManager()), - AllocateGVsWithCode(GVsWithCode), isAlreadyCodeGenerating(false) { - setDataLayout(TM.getDataLayout()); - - jitstate = new JITState(M); - - // Initialize JCE - JCE = createEmitter(*this, JMM, TM); - - // Register in global list of all JITs. - AllJits->Add(this); - - // Add target data - MutexGuard locked(lock); - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory that - // may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); -} - -JIT::~JIT() { - // Cleanup. - AllJits->Remove(this); - delete jitstate; - delete JCE; - // JMM is a ownership of JCE, so we no need delete JMM here. - delete &TM; -} - -/// addModule - Add a new Module to the JIT. If we previously removed the last -/// Module, we need re-initialize jitstate with a valid Module. -void JIT::addModule(Module *M) { - MutexGuard locked(lock); - - if (Modules.empty()) { - assert(!jitstate && "jitstate should be NULL if Modules vector is empty!"); - - jitstate = new JITState(M); - - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory - // that may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); - } - - ExecutionEngine::addModule(M); -} - -/// removeModule - If we are removing the last Module, invalidate the jitstate -/// since the PassManager it contains references a released Module. -bool JIT::removeModule(Module *M) { - bool result = ExecutionEngine::removeModule(M); - - MutexGuard locked(lock); - - if (jitstate && jitstate->getModule() == M) { - delete jitstate; - jitstate = nullptr; - } - - if (!jitstate && !Modules.empty()) { - jitstate = new JITState(Modules[0]); - - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory - // that may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); - } - return result; -} - -/// run - Start execution with the specified function and arguments. -/// -GenericValue JIT::runFunction(Function *F, - const std::vector<GenericValue> &ArgValues) { - assert(F && "Function *F was null at entry to run()"); - - void *FPtr = getPointerToFunction(F); - assert(FPtr && "Pointer to fn's code was null after getPointerToFunction"); - FunctionType *FTy = F->getFunctionType(); - Type *RetTy = FTy->getReturnType(); - - assert((FTy->getNumParams() == ArgValues.size() || - (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) && - "Wrong number of arguments passed into function!"); - assert(FTy->getNumParams() == ArgValues.size() && - "This doesn't support passing arguments through varargs (yet)!"); - - // Handle some common cases first. These cases correspond to common `main' - // prototypes. - if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) { - switch (ArgValues.size()) { - case 3: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy() && - FTy->getParamType(2)->isPointerTy()) { - int (*PF)(int, char **, const char **) = - (int(*)(int, char **, const char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]), - (const char **)GVTOP(ArgValues[2]))); - return rv; - } - break; - case 2: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy()) { - int (*PF)(int, char **) = (int(*)(int, char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]))); - return rv; - } - break; - case 1: - if (FTy->getParamType(0)->isIntegerTy(32)) { - GenericValue rv; - int (*PF)(int) = (int(*)(int))(intptr_t)FPtr; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue())); - return rv; - } - if (FTy->getParamType(0)->isPointerTy()) { - GenericValue rv; - int (*PF)(char *) = (int(*)(char *))(intptr_t)FPtr; - rv.IntVal = APInt(32, PF((char*)GVTOP(ArgValues[0]))); - return rv; - } - break; - } - } - - // Handle cases where no arguments are passed first. - if (ArgValues.empty()) { - GenericValue rv; - switch (RetTy->getTypeID()) { - default: llvm_unreachable("Unknown return type for function call!"); - case Type::IntegerTyID: { - unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth(); - if (BitWidth == 1) - rv.IntVal = APInt(BitWidth, ((bool(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 8) - rv.IntVal = APInt(BitWidth, ((char(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 16) - rv.IntVal = APInt(BitWidth, ((short(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 32) - rv.IntVal = APInt(BitWidth, ((int(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 64) - rv.IntVal = APInt(BitWidth, ((int64_t(*)())(intptr_t)FPtr)()); - else - llvm_unreachable("Integer types > 64 bits not supported"); - return rv; - } - case Type::VoidTyID: - rv.IntVal = APInt(32, ((int(*)())(intptr_t)FPtr)()); - return rv; - case Type::FloatTyID: - rv.FloatVal = ((float(*)())(intptr_t)FPtr)(); - return rv; - case Type::DoubleTyID: - rv.DoubleVal = ((double(*)())(intptr_t)FPtr)(); - return rv; - case Type::X86_FP80TyID: - case Type::FP128TyID: - case Type::PPC_FP128TyID: - llvm_unreachable("long double not supported yet"); - case Type::PointerTyID: - return PTOGV(((void*(*)())(intptr_t)FPtr)()); - } - } - - // Okay, this is not one of our quick and easy cases. Because we don't have a - // full FFI, we have to codegen a nullary stub function that just calls the - // function we are interested in, passing in constants for all of the - // arguments. Make this function and return. - - // First, create the function. - FunctionType *STy=FunctionType::get(RetTy, false); - Function *Stub = Function::Create(STy, Function::InternalLinkage, "", - F->getParent()); - - // Insert a basic block. - BasicBlock *StubBB = BasicBlock::Create(F->getContext(), "", Stub); - - // Convert all of the GenericValue arguments over to constants. Note that we - // currently don't support varargs. - SmallVector<Value*, 8> Args; - for (unsigned i = 0, e = ArgValues.size(); i != e; ++i) { - Constant *C = nullptr; - Type *ArgTy = FTy->getParamType(i); - const GenericValue &AV = ArgValues[i]; - switch (ArgTy->getTypeID()) { - default: llvm_unreachable("Unknown argument type for function call!"); - case Type::IntegerTyID: - C = ConstantInt::get(F->getContext(), AV.IntVal); - break; - case Type::FloatTyID: - C = ConstantFP::get(F->getContext(), APFloat(AV.FloatVal)); - break; - case Type::DoubleTyID: - C = ConstantFP::get(F->getContext(), APFloat(AV.DoubleVal)); - break; - case Type::PPC_FP128TyID: - case Type::X86_FP80TyID: - case Type::FP128TyID: - C = ConstantFP::get(F->getContext(), APFloat(ArgTy->getFltSemantics(), - AV.IntVal)); - break; - case Type::PointerTyID: - void *ArgPtr = GVTOP(AV); - if (sizeof(void*) == 4) - C = ConstantInt::get(Type::getInt32Ty(F->getContext()), - (int)(intptr_t)ArgPtr); - else - C = ConstantInt::get(Type::getInt64Ty(F->getContext()), - (intptr_t)ArgPtr); - // Cast the integer to pointer - C = ConstantExpr::getIntToPtr(C, ArgTy); - break; - } - Args.push_back(C); - } - - CallInst *TheCall = CallInst::Create(F, Args, "", StubBB); - TheCall->setCallingConv(F->getCallingConv()); - TheCall->setTailCall(); - if (!TheCall->getType()->isVoidTy()) - // Return result of the call. - ReturnInst::Create(F->getContext(), TheCall, StubBB); - else - ReturnInst::Create(F->getContext(), StubBB); // Just return void. - - // Finally, call our nullary stub function. - GenericValue Result = runFunction(Stub, std::vector<GenericValue>()); - // Erase it, since no other function can have a reference to it. - Stub->eraseFromParent(); - // And return the result. - return Result; -} - -void JIT::RegisterJITEventListener(JITEventListener *L) { - if (!L) - return; - MutexGuard locked(lock); - EventListeners.push_back(L); -} -void JIT::UnregisterJITEventListener(JITEventListener *L) { - if (!L) - return; - MutexGuard locked(lock); - std::vector<JITEventListener*>::reverse_iterator I= - std::find(EventListeners.rbegin(), EventListeners.rend(), L); - if (I != EventListeners.rend()) { - std::swap(*I, EventListeners.back()); - EventListeners.pop_back(); - } -} -void JIT::NotifyFunctionEmitted( - const Function &F, - void *Code, size_t Size, - const JITEvent_EmittedFunctionDetails &Details) { - MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFunctionEmitted(F, Code, Size, Details); - } -} - -void JIT::NotifyFreeingMachineCode(void *OldPtr) { - MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFreeingMachineCode(OldPtr); - } -} - -/// runJITOnFunction - Run the FunctionPassManager full of -/// just-in-time compilation passes on F, hopefully filling in -/// GlobalAddress[F] with the address of F's machine code. -/// -void JIT::runJITOnFunction(Function *F, MachineCodeInfo *MCI) { - MutexGuard locked(lock); - - class MCIListener : public JITEventListener { - MachineCodeInfo *const MCI; - public: - MCIListener(MachineCodeInfo *mci) : MCI(mci) {} - void NotifyFunctionEmitted(const Function &, void *Code, size_t Size, - const EmittedFunctionDetails &) override { - MCI->setAddress(Code); - MCI->setSize(Size); - } - }; - MCIListener MCIL(MCI); - if (MCI) - RegisterJITEventListener(&MCIL); - - runJITOnFunctionUnlocked(F); - - if (MCI) - UnregisterJITEventListener(&MCIL); -} - -void JIT::runJITOnFunctionUnlocked(Function *F) { - assert(!isAlreadyCodeGenerating && "Error: Recursive compilation detected!"); - - jitTheFunctionUnlocked(F); - - // If the function referred to another function that had not yet been - // read from bitcode, and we are jitting non-lazily, emit it now. - while (!jitstate->getPendingFunctions().empty()) { - Function *PF = jitstate->getPendingFunctions().back(); - jitstate->getPendingFunctions().pop_back(); - - assert(!PF->hasAvailableExternallyLinkage() && - "Externally-defined function should not be in pending list."); - - jitTheFunctionUnlocked(PF); - - // Now that the function has been jitted, ask the JITEmitter to rewrite - // the stub with real address of the function. - updateFunctionStubUnlocked(PF); - } -} - -void JIT::jitTheFunctionUnlocked(Function *F) { - isAlreadyCodeGenerating = true; - jitstate->getPM().run(*F); - isAlreadyCodeGenerating = false; - - // clear basic block addresses after this function is done - getBasicBlockAddressMap().clear(); -} - -/// getPointerToFunction - This method is used to get the address of the -/// specified function, compiling it if necessary. -/// -void *JIT::getPointerToFunction(Function *F) { - - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; // Check if function already code gen'd - - MutexGuard locked(lock); - - // Now that this thread owns the lock, make sure we read in the function if it - // exists in this Module. - std::string ErrorMsg; - if (F->Materialize(&ErrorMsg)) { - report_fatal_error("Error reading function '" + F->getName()+ - "' from bitcode file: " + ErrorMsg); - } - - // ... and check if another thread has already code gen'd the function. - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; - - if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) { - bool AbortOnFailure = !F->hasExternalWeakLinkage(); - void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure); - addGlobalMapping(F, Addr); - return Addr; - } - - runJITOnFunctionUnlocked(F); - - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr && "Code generation didn't add function to GlobalAddress table!"); - return Addr; -} - -void JIT::addPointerToBasicBlock(const BasicBlock *BB, void *Addr) { - MutexGuard locked(lock); - - BasicBlockAddressMapTy::iterator I = - getBasicBlockAddressMap().find(BB); - if (I == getBasicBlockAddressMap().end()) { - getBasicBlockAddressMap()[BB] = Addr; - } else { - // ignore repeats: some BBs can be split into few MBBs? - } -} - -void JIT::clearPointerToBasicBlock(const BasicBlock *BB) { - MutexGuard locked(lock); - getBasicBlockAddressMap().erase(BB); -} - -void *JIT::getPointerToBasicBlock(BasicBlock *BB) { - // make sure it's function is compiled by JIT - (void)getPointerToFunction(BB->getParent()); - - // resolve basic block address - MutexGuard locked(lock); - - BasicBlockAddressMapTy::iterator I = - getBasicBlockAddressMap().find(BB); - if (I != getBasicBlockAddressMap().end()) { - return I->second; - } else { - llvm_unreachable("JIT does not have BB address for address-of-label, was" - " it eliminated by optimizer?"); - } -} - -void *JIT::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure){ - if (!isSymbolSearchingDisabled()) { - void *ptr = JMM->getPointerToNamedFunction(Name, false); - if (ptr) - return ptr; - } - - /// If a LazyFunctionCreator is installed, use it to get/create the function. - if (LazyFunctionCreator) - if (void *RP = LazyFunctionCreator(Name)) - return RP; - - if (AbortOnFailure) { - report_fatal_error("Program used external function '"+Name+ - "' which could not be resolved!"); - } - return nullptr; -} - - -/// getOrEmitGlobalVariable - Return the address of the specified global -/// variable, possibly emitting it to memory if needed. This is used by the -/// Emitter. -void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) { - MutexGuard locked(lock); - - void *Ptr = getPointerToGlobalIfAvailable(GV); - if (Ptr) return Ptr; - - // If the global is external, just remember the address. - if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) { -#if HAVE___DSO_HANDLE - if (GV->getName() == "__dso_handle") - return (void*)&__dso_handle; -#endif - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(GV->getName()); - if (!Ptr) { - report_fatal_error("Could not resolve external global address: " - +GV->getName()); - } - addGlobalMapping(GV, Ptr); - } else { - // If the global hasn't been emitted to memory yet, allocate space and - // emit it into memory. - Ptr = getMemoryForGV(GV); - addGlobalMapping(GV, Ptr); - EmitGlobalVariable(GV); // Initialize the variable. - } - return Ptr; -} - -/// recompileAndRelinkFunction - This method is used to force a function -/// which has already been compiled, to be compiled again, possibly -/// after it has been modified. Then the entry to the old copy is overwritten -/// with a branch to the new copy. If there was no old copy, this acts -/// just like JIT::getPointerToFunction(). -/// -void *JIT::recompileAndRelinkFunction(Function *F) { - void *OldAddr = getPointerToGlobalIfAvailable(F); - - // If it's not already compiled there is no reason to patch it up. - if (!OldAddr) return getPointerToFunction(F); - - // Delete the old function mapping. - addGlobalMapping(F, nullptr); - - // Recodegen the function - runJITOnFunction(F); - - // Update state, forward the old function to the new function. - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr && "Code generation didn't add function to GlobalAddress table!"); - TJI.replaceMachineCodeForFunction(OldAddr, Addr); - return Addr; -} - -/// getMemoryForGV - This method abstracts memory allocation of global -/// variable so that the JIT can allocate thread local variables depending -/// on the target. -/// -char* JIT::getMemoryForGV(const GlobalVariable* GV) { - char *Ptr; - - // GlobalVariable's which are not "constant" will cause trouble in a server - // situation. It's returned in the same block of memory as code which may - // not be writable. - if (isGVCompilationDisabled() && !GV->isConstant()) { - report_fatal_error("Compilation of non-internal GlobalValue is disabled!"); - } - - // Some applications require globals and code to live together, so they may - // be allocated into the same buffer, but in general globals are allocated - // through the memory manager which puts them near the code but not in the - // same buffer. - Type *GlobalType = GV->getType()->getElementType(); - size_t S = getDataLayout()->getTypeAllocSize(GlobalType); - size_t A = getDataLayout()->getPreferredAlignment(GV); - if (GV->isThreadLocal()) { - MutexGuard locked(lock); - Ptr = TJI.allocateThreadLocalMemory(S); - } else if (TJI.allocateSeparateGVMemory()) { - if (A <= 8) { - Ptr = (char*)malloc(S); - } else { - // Allocate S+A bytes of memory, then use an aligned pointer within that - // space. - Ptr = (char*)malloc(S+A); - unsigned MisAligned = ((intptr_t)Ptr & (A-1)); - Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0); - } - } else if (AllocateGVsWithCode) { - Ptr = (char*)JCE->allocateSpace(S, A); - } else { - Ptr = (char*)JCE->allocateGlobal(S, A); - } - return Ptr; -} - -void JIT::addPendingFunction(Function *F) { - MutexGuard locked(lock); - jitstate->getPendingFunctions().push_back(F); -} - - -JITEventListener::~JITEventListener() {} diff --git a/lib/ExecutionEngine/JIT/JIT.h b/lib/ExecutionEngine/JIT/JIT.h deleted file mode 100644 index 69a7c3670a87..000000000000 --- a/lib/ExecutionEngine/JIT/JIT.h +++ /dev/null @@ -1,229 +0,0 @@ -//===-- JIT.h - Class definition for the JIT --------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the top-level JIT data structure. -// -//===----------------------------------------------------------------------===// - -#ifndef JIT_H -#define JIT_H - -#include "llvm/ExecutionEngine/ExecutionEngine.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/PassManager.h" - -namespace llvm { - -class Function; -struct JITEvent_EmittedFunctionDetails; -class MachineCodeEmitter; -class MachineCodeInfo; -class TargetJITInfo; -class TargetMachine; - -class JITState { -private: - FunctionPassManager PM; // Passes to compile a function - Module *M; // Module used to create the PM - - /// PendingFunctions - Functions which have not been code generated yet, but - /// were called from a function being code generated. - std::vector<AssertingVH<Function> > PendingFunctions; - -public: - explicit JITState(Module *M) : PM(M), M(M) {} - - FunctionPassManager &getPM() { - return PM; - } - - Module *getModule() const { return M; } - std::vector<AssertingVH<Function> > &getPendingFunctions() { - return PendingFunctions; - } -}; - - -class JIT : public ExecutionEngine { - /// types - typedef ValueMap<const BasicBlock *, void *> - BasicBlockAddressMapTy; - /// data - TargetMachine &TM; // The current target we are compiling to - TargetJITInfo &TJI; // The JITInfo for the target we are compiling to - JITCodeEmitter *JCE; // JCE object - JITMemoryManager *JMM; - std::vector<JITEventListener*> EventListeners; - - /// AllocateGVsWithCode - Some applications require that global variables and - /// code be allocated into the same region of memory, in which case this flag - /// should be set to true. Doing so breaks freeMachineCodeForFunction. - bool AllocateGVsWithCode; - - /// True while the JIT is generating code. Used to assert against recursive - /// entry. - bool isAlreadyCodeGenerating; - - JITState *jitstate; - - /// BasicBlockAddressMap - A mapping between LLVM basic blocks and their - /// actualized version, only filled for basic blocks that have their address - /// taken. - BasicBlockAddressMapTy BasicBlockAddressMap; - - - JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji, - JITMemoryManager *JMM, bool AllocateGVsWithCode); -public: - ~JIT(); - - static void Register() { - JITCtor = createJIT; - } - - /// getJITInfo - Return the target JIT information structure. - /// - TargetJITInfo &getJITInfo() const { return TJI; } - - /// create - Create an return a new JIT compiler if there is one available - /// for the current target. Otherwise, return null. - /// - static ExecutionEngine *create(Module *M, - std::string *Err, - JITMemoryManager *JMM, - CodeGenOpt::Level OptLevel = - CodeGenOpt::Default, - bool GVsWithCode = true, - Reloc::Model RM = Reloc::Default, - CodeModel::Model CMM = CodeModel::JITDefault) { - return ExecutionEngine::createJIT(M, Err, JMM, OptLevel, GVsWithCode, - RM, CMM); - } - - void addModule(Module *M) override; - - /// removeModule - Remove a Module from the list of modules. Returns true if - /// M is found. - bool removeModule(Module *M) override; - - /// runFunction - Start execution with the specified function and arguments. - /// - GenericValue runFunction(Function *F, - const std::vector<GenericValue> &ArgValues) override; - - /// getPointerToNamedFunction - This method returns the address of the - /// specified function by using the MemoryManager. As such it is only - /// useful for resolving library symbols, not code generated symbols. - /// - /// If AbortOnFailure is false and no function with the given name is - /// found, this function silently returns a null pointer. Otherwise, - /// it prints a message to stderr and aborts. - /// - void *getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure = true) override; - - // CompilationCallback - Invoked the first time that a call site is found, - // which causes lazy compilation of the target function. - // - static void CompilationCallback(); - - /// getPointerToFunction - This returns the address of the specified function, - /// compiling it if necessary. - /// - void *getPointerToFunction(Function *F) override; - - /// addPointerToBasicBlock - Adds address of the specific basic block. - void addPointerToBasicBlock(const BasicBlock *BB, void *Addr); - - /// clearPointerToBasicBlock - Removes address of specific basic block. - void clearPointerToBasicBlock(const BasicBlock *BB); - - /// getPointerToBasicBlock - This returns the address of the specified basic - /// block, assuming function is compiled. - void *getPointerToBasicBlock(BasicBlock *BB) override; - - /// getOrEmitGlobalVariable - Return the address of the specified global - /// variable, possibly emitting it to memory if needed. This is used by the - /// Emitter. - void *getOrEmitGlobalVariable(const GlobalVariable *GV) override; - - /// getPointerToFunctionOrStub - If the specified function has been - /// code-gen'd, return a pointer to the function. If not, compile it, or use - /// a stub to implement lazy compilation if available. - /// - void *getPointerToFunctionOrStub(Function *F) override; - - /// recompileAndRelinkFunction - This method is used to force a function - /// which has already been compiled, to be compiled again, possibly - /// after it has been modified. Then the entry to the old copy is overwritten - /// with a branch to the new copy. If there was no old copy, this acts - /// just like JIT::getPointerToFunction(). - /// - void *recompileAndRelinkFunction(Function *F) override; - - /// freeMachineCodeForFunction - deallocate memory used to code-generate this - /// Function. - /// - void freeMachineCodeForFunction(Function *F) override; - - /// addPendingFunction - while jitting non-lazily, a called but non-codegen'd - /// function was encountered. Add it to a pending list to be processed after - /// the current function. - /// - void addPendingFunction(Function *F); - - /// getCodeEmitter - Return the code emitter this JIT is emitting into. - /// - JITCodeEmitter *getCodeEmitter() const { return JCE; } - - static ExecutionEngine *createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM); - - // Run the JIT on F and return information about the generated code - void runJITOnFunction(Function *F, MachineCodeInfo *MCI = nullptr) override; - - void RegisterJITEventListener(JITEventListener *L) override; - void UnregisterJITEventListener(JITEventListener *L) override; - - TargetMachine *getTargetMachine() override { return &TM; } - - /// These functions correspond to the methods on JITEventListener. They - /// iterate over the registered listeners and call the corresponding method on - /// each. - void NotifyFunctionEmitted( - const Function &F, void *Code, size_t Size, - const JITEvent_EmittedFunctionDetails &Details); - void NotifyFreeingMachineCode(void *OldPtr); - - BasicBlockAddressMapTy & - getBasicBlockAddressMap() { - return BasicBlockAddressMap; - } - - -private: - static JITCodeEmitter *createEmitter(JIT &J, JITMemoryManager *JMM, - TargetMachine &tm); - void runJITOnFunctionUnlocked(Function *F); - void updateFunctionStubUnlocked(Function *F); - void jitTheFunctionUnlocked(Function *F); - -protected: - - /// getMemoryforGV - Allocate memory for a global variable. - char* getMemoryForGV(const GlobalVariable* GV) override; - -}; - -} // End llvm namespace - -#endif diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp deleted file mode 100644 index 2ba1f8695d7c..000000000000 --- a/lib/ExecutionEngine/JIT/JITEmitter.cpp +++ /dev/null @@ -1,1249 +0,0 @@ -//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines a MachineCodeEmitter object that is used by the JIT to -// write machine code to memory and remember where relocatable values are. -// -//===----------------------------------------------------------------------===// - -#include "JIT.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/CodeGen/MachineConstantPool.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" -#include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/CodeGen/MachineRelocation.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Operator.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/IR/ValueMap.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/MutexGuard.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include <algorithm> -#ifndef NDEBUG -#include <iomanip> -#endif -using namespace llvm; - -#define DEBUG_TYPE "jit" - -STATISTIC(NumBytes, "Number of bytes of machine code compiled"); -STATISTIC(NumRelos, "Number of relocations applied"); -STATISTIC(NumRetries, "Number of retries with more memory"); - - -// A declaration may stop being a declaration once it's fully read from bitcode. -// This function returns true if F is fully read and is still a declaration. -static bool isNonGhostDeclaration(const Function *F) { - return F->isDeclaration() && !F->isMaterializable(); -} - -//===----------------------------------------------------------------------===// -// JIT lazy compilation code. -// -namespace { - class JITEmitter; - class JITResolverState; - - template<typename ValueTy> - struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { - typedef JITResolverState *ExtraData; - static void onRAUW(JITResolverState *, Value *Old, Value *New) { - llvm_unreachable("The JIT doesn't know how to handle a" - " RAUW on a value it has emitted."); - } - }; - - struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { - typedef JITResolverState *ExtraData; - static void onDelete(JITResolverState *JRS, Function *F); - }; - - class JITResolverState { - public: - typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > - FunctionToLazyStubMapTy; - typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; - typedef ValueMap<Function *, SmallPtrSet<void*, 1>, - CallSiteValueMapConfig> FunctionToCallSitesMapTy; - typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; - private: - /// FunctionToLazyStubMap - Keep track of the lazy stub created for a - /// particular function so that we can reuse them if necessary. - FunctionToLazyStubMapTy FunctionToLazyStubMap; - - /// CallSiteToFunctionMap - Keep track of the function that each lazy call - /// site corresponds to, and vice versa. - CallSiteToFunctionMapTy CallSiteToFunctionMap; - FunctionToCallSitesMapTy FunctionToCallSitesMap; - - /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a - /// particular GlobalVariable so that we can reuse them if necessary. - GlobalToIndirectSymMapTy GlobalToIndirectSymMap; - -#ifndef NDEBUG - /// Instance of the JIT this ResolverState serves. - JIT *TheJIT; -#endif - - public: - JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), - FunctionToCallSitesMap(this) { -#ifndef NDEBUG - TheJIT = jit; -#endif - } - - FunctionToLazyStubMapTy& getFunctionToLazyStubMap() { - return FunctionToLazyStubMap; - } - - GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap() { - return GlobalToIndirectSymMap; - } - - std::pair<void *, Function *> LookupFunctionFromCallSite( - void *CallSite) const { - // The address given to us for the stub may not be exactly right, it - // might be a little bit after the stub. As such, use upper_bound to - // find it. - CallSiteToFunctionMapTy::const_iterator I = - CallSiteToFunctionMap.upper_bound(CallSite); - assert(I != CallSiteToFunctionMap.begin() && - "This is not a known call site!"); - --I; - return *I; - } - - void AddCallSite(void *CallSite, Function *F) { - bool Inserted = CallSiteToFunctionMap.insert( - std::make_pair(CallSite, F)).second; - (void)Inserted; - assert(Inserted && "Pair was already in CallSiteToFunctionMap"); - FunctionToCallSitesMap[F].insert(CallSite); - } - - void EraseAllCallSitesForPrelocked(Function *F); - - // Erases _all_ call sites regardless of their function. This is used to - // unregister the stub addresses from the StubToResolverMap in - // ~JITResolver(). - void EraseAllCallSitesPrelocked(); - }; - - /// JITResolver - Keep track of, and resolve, call sites for functions that - /// have not yet been compiled. - class JITResolver { - typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; - typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; - typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; - - /// LazyResolverFn - The target lazy resolver function that we actually - /// rewrite instructions to use. - TargetJITInfo::LazyResolverFn LazyResolverFn; - - JITResolverState state; - - /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap - /// for external functions. TODO: Of course, external functions don't need - /// a lazy stub. It's actually here to make it more likely that far calls - /// succeed, but no single stub can guarantee that. I'll remove this in a - /// subsequent checkin when I actually fix far calls. - std::map<void*, void*> ExternalFnToStubMap; - - /// revGOTMap - map addresses to indexes in the GOT - std::map<void*, unsigned> revGOTMap; - unsigned nextGOTIndex; - - JITEmitter &JE; - - /// Instance of JIT corresponding to this Resolver. - JIT *TheJIT; - - public: - explicit JITResolver(JIT &jit, JITEmitter &je) - : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { - LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); - } - - ~JITResolver(); - - /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's - /// lazy-compilation stub if it has already been created. - void *getLazyFunctionStubIfAvailable(Function *F); - - /// getLazyFunctionStub - This returns a pointer to a function's - /// lazy-compilation stub, creating one on demand as needed. - void *getLazyFunctionStub(Function *F); - - /// getExternalFunctionStub - Return a stub for the function at the - /// specified address, created lazily on demand. - void *getExternalFunctionStub(void *FnAddr); - - /// getGlobalValueIndirectSym - Return an indirect symbol containing the - /// specified GV address. - void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); - - /// getGOTIndexForAddress - Return a new or existing index in the GOT for - /// an address. This function only manages slots, it does not manage the - /// contents of the slots or the memory associated with the GOT. - unsigned getGOTIndexForAddr(void *addr); - - /// JITCompilerFn - This function is called to resolve a stub to a compiled - /// address. If the LLVM Function corresponding to the stub has not yet - /// been compiled, this function compiles it first. - static void *JITCompilerFn(void *Stub); - }; - - class StubToResolverMapTy { - /// Map a stub address to a specific instance of a JITResolver so that - /// lazily-compiled functions can find the right resolver to use. - /// - /// Guarded by Lock. - std::map<void*, JITResolver*> Map; - - /// Guards Map from concurrent accesses. - mutable sys::Mutex Lock; - - public: - /// Registers a Stub to be resolved by Resolver. - void RegisterStubResolver(void *Stub, JITResolver *Resolver) { - MutexGuard guard(Lock); - Map.insert(std::make_pair(Stub, Resolver)); - } - /// Unregisters the Stub when it's invalidated. - void UnregisterStubResolver(void *Stub) { - MutexGuard guard(Lock); - Map.erase(Stub); - } - /// Returns the JITResolver instance that owns the Stub. - JITResolver *getResolverFromStub(void *Stub) const { - MutexGuard guard(Lock); - // The address given to us for the stub may not be exactly right, it might - // be a little bit after the stub. As such, use upper_bound to find it. - // This is the same trick as in LookupFunctionFromCallSite from - // JITResolverState. - std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); - assert(I != Map.begin() && "This is not a known stub!"); - --I; - return I->second; - } - /// True if any stubs refer to the given resolver. Only used in an assert(). - /// O(N) - bool ResolverHasStubs(JITResolver* Resolver) const { - MutexGuard guard(Lock); - for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), - E = Map.end(); I != E; ++I) { - if (I->second == Resolver) - return true; - } - return false; - } - }; - /// This needs to be static so that a lazy call stub can access it with no - /// context except the address of the stub. - ManagedStatic<StubToResolverMapTy> StubToResolverMap; - - /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is - /// used to output functions to memory for execution. - class JITEmitter : public JITCodeEmitter { - JITMemoryManager *MemMgr; - - // When outputting a function stub in the context of some other function, we - // save BufferBegin/BufferEnd/CurBufferPtr here. - uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; - - // When reattempting to JIT a function after running out of space, we store - // the estimated size of the function we're trying to JIT here, so we can - // ask the memory manager for at least this much space. When we - // successfully emit the function, we reset this back to zero. - uintptr_t SizeEstimate; - - /// Relocations - These are the relocations that the function needs, as - /// emitted. - std::vector<MachineRelocation> Relocations; - - /// MBBLocations - This vector is a mapping from MBB ID's to their address. - /// It is filled in by the StartMachineBasicBlock callback and queried by - /// the getMachineBasicBlockAddress callback. - std::vector<uintptr_t> MBBLocations; - - /// ConstantPool - The constant pool for the current function. - /// - MachineConstantPool *ConstantPool; - - /// ConstantPoolBase - A pointer to the first entry in the constant pool. - /// - void *ConstantPoolBase; - - /// ConstPoolAddresses - Addresses of individual constant pool entries. - /// - SmallVector<uintptr_t, 8> ConstPoolAddresses; - - /// JumpTable - The jump tables for the current function. - /// - MachineJumpTableInfo *JumpTable; - - /// JumpTableBase - A pointer to the first entry in the jump table. - /// - void *JumpTableBase; - - /// Resolver - This contains info about the currently resolved functions. - JITResolver Resolver; - - /// LabelLocations - This vector is a mapping from Label ID's to their - /// address. - DenseMap<MCSymbol*, uintptr_t> LabelLocations; - - /// MMI - Machine module info for exception informations - MachineModuleInfo* MMI; - - // CurFn - The llvm function being emitted. Only valid during - // finishFunction(). - const Function *CurFn; - - /// Information about emitted code, which is passed to the - /// JITEventListeners. This is reset in startFunction and used in - /// finishFunction. - JITEvent_EmittedFunctionDetails EmissionDetails; - - struct EmittedCode { - void *FunctionBody; // Beginning of the function's allocation. - void *Code; // The address the function's code actually starts at. - void *ExceptionTable; - EmittedCode() : FunctionBody(nullptr), Code(nullptr), - ExceptionTable(nullptr) {} - }; - struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { - typedef JITEmitter *ExtraData; - static void onDelete(JITEmitter *, const Function*); - static void onRAUW(JITEmitter *, const Function*, const Function*); - }; - ValueMap<const Function *, EmittedCode, - EmittedFunctionConfig> EmittedFunctions; - - DebugLoc PrevDL; - - /// Instance of the JIT - JIT *TheJIT; - - public: - JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) - : SizeEstimate(0), Resolver(jit, *this), MMI(nullptr), CurFn(nullptr), - EmittedFunctions(this), TheJIT(&jit) { - MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); - if (jit.getJITInfo().needsGOT()) { - MemMgr->AllocateGOT(); - DEBUG(dbgs() << "JIT is managing a GOT\n"); - } - - } - ~JITEmitter() { - delete MemMgr; - } - - JITResolver &getJITResolver() { return Resolver; } - - void startFunction(MachineFunction &F) override; - bool finishFunction(MachineFunction &F) override; - - void emitConstantPool(MachineConstantPool *MCP); - void initJumpTableInfo(MachineJumpTableInfo *MJTI); - void emitJumpTableInfo(MachineJumpTableInfo *MJTI); - - void startGVStub(const GlobalValue* GV, - unsigned StubSize, unsigned Alignment = 1); - void startGVStub(void *Buffer, unsigned StubSize); - void finishGVStub(); - void *allocIndirectGV(const GlobalValue *GV, const uint8_t *Buffer, - size_t Size, unsigned Alignment) override; - - /// allocateSpace - Reserves space in the current block if any, or - /// allocate a new one of the given size. - void *allocateSpace(uintptr_t Size, unsigned Alignment) override; - - /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, - /// this method does not allocate memory in the current output buffer, - /// because a global may live longer than the current function. - void *allocateGlobal(uintptr_t Size, unsigned Alignment) override; - - void addRelocation(const MachineRelocation &MR) override { - Relocations.push_back(MR); - } - - void StartMachineBasicBlock(MachineBasicBlock *MBB) override { - if (MBBLocations.size() <= (unsigned)MBB->getNumber()) - MBBLocations.resize((MBB->getNumber()+1)*2); - MBBLocations[MBB->getNumber()] = getCurrentPCValue(); - if (MBB->hasAddressTaken()) - TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), - (void*)getCurrentPCValue()); - DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" - << (void*) getCurrentPCValue() << "]\n"); - } - - uintptr_t getConstantPoolEntryAddress(unsigned Entry) const override; - uintptr_t getJumpTableEntryAddress(unsigned Entry) const override; - - uintptr_t - getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override { - assert(MBBLocations.size() > (unsigned)MBB->getNumber() && - MBBLocations[MBB->getNumber()] && "MBB not emitted!"); - return MBBLocations[MBB->getNumber()]; - } - - /// retryWithMoreMemory - Log a retry and deallocate all memory for the - /// given function. Increase the minimum allocation size so that we get - /// more memory next time. - void retryWithMoreMemory(MachineFunction &F); - - /// deallocateMemForFunction - Deallocate all memory for the specified - /// function body. - void deallocateMemForFunction(const Function *F); - - void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) override; - - void emitLabel(MCSymbol *Label) override { - LabelLocations[Label] = getCurrentPCValue(); - } - - DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() override { - return &LabelLocations; - } - - uintptr_t getLabelAddress(MCSymbol *Label) const override { - assert(LabelLocations.count(Label) && "Label not emitted!"); - return LabelLocations.find(Label)->second; - } - - void setModuleInfo(MachineModuleInfo* Info) override { - MMI = Info; - } - - private: - void *getPointerToGlobal(GlobalValue *GV, void *Reference, - bool MayNeedFarStub); - void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); - }; -} - -void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { - JRS->EraseAllCallSitesForPrelocked(F); -} - -void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { - FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); - if (F2C == FunctionToCallSitesMap.end()) - return; - StubToResolverMapTy &S2RMap = *StubToResolverMap; - for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), - E = F2C->second.end(); I != E; ++I) { - S2RMap.UnregisterStubResolver(*I); - bool Erased = CallSiteToFunctionMap.erase(*I); - (void)Erased; - assert(Erased && "Missing call site->function mapping"); - } - FunctionToCallSitesMap.erase(F2C); -} - -void JITResolverState::EraseAllCallSitesPrelocked() { - StubToResolverMapTy &S2RMap = *StubToResolverMap; - for (CallSiteToFunctionMapTy::const_iterator - I = CallSiteToFunctionMap.begin(), - E = CallSiteToFunctionMap.end(); I != E; ++I) { - S2RMap.UnregisterStubResolver(I->first); - } - CallSiteToFunctionMap.clear(); - FunctionToCallSitesMap.clear(); -} - -JITResolver::~JITResolver() { - // No need to lock because we're in the destructor, and state isn't shared. - state.EraseAllCallSitesPrelocked(); - assert(!StubToResolverMap->ResolverHasStubs(this) && - "Resolver destroyed with stubs still alive."); -} - -/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub -/// if it has already been created. -void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { - MutexGuard locked(TheJIT->lock); - - // If we already have a stub for this function, recycle it. - return state.getFunctionToLazyStubMap().lookup(F); -} - -/// getFunctionStub - This returns a pointer to a function stub, creating -/// one on demand as needed. -void *JITResolver::getLazyFunctionStub(Function *F) { - MutexGuard locked(TheJIT->lock); - - // If we already have a lazy stub for this function, recycle it. - void *&Stub = state.getFunctionToLazyStubMap()[F]; - if (Stub) return Stub; - - // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we - // must resolve the symbol now. - void *Actual = TheJIT->isCompilingLazily() - ? (void *)(intptr_t)LazyResolverFn : (void *)nullptr; - - // If this is an external declaration, attempt to resolve the address now - // to place in the stub. - if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { - Actual = TheJIT->getPointerToFunction(F); - - // If we resolved the symbol to a null address (eg. a weak external) - // don't emit a stub. Return a null pointer to the application. - if (!Actual) return nullptr; - } - - TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); - JE.startGVStub(F, SL.Size, SL.Alignment); - // Codegen a new stub, calling the lazy resolver or the actual address of the - // external function, if it was resolved. - Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); - JE.finishGVStub(); - - if (Actual != (void*)(intptr_t)LazyResolverFn) { - // If we are getting the stub for an external function, we really want the - // address of the stub in the GlobalAddressMap for the JIT, not the address - // of the external function. - TheJIT->updateGlobalMapping(F, Stub); - } - - DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" - << F->getName() << "'\n"); - - if (TheJIT->isCompilingLazily()) { - // Register this JITResolver as the one corresponding to this call site so - // JITCompilerFn will be able to find it. - StubToResolverMap->RegisterStubResolver(Stub, this); - - // Finally, keep track of the stub-to-Function mapping so that the - // JITCompilerFn knows which function to compile! - state.AddCallSite(Stub, F); - } else if (!Actual) { - // If we are JIT'ing non-lazily but need to call a function that does not - // exist yet, add it to the JIT's work list so that we can fill in the - // stub address later. - assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && - "'Actual' should have been set above."); - TheJIT->addPendingFunction(F); - } - - return Stub; -} - -/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified -/// GV address. -void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { - MutexGuard locked(TheJIT->lock); - - // If we already have a stub for this global variable, recycle it. - void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV]; - if (IndirectSym) return IndirectSym; - - // Otherwise, codegen a new indirect symbol. - IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, - JE); - - DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym - << "] for GV '" << GV->getName() << "'\n"); - - return IndirectSym; -} - -/// getExternalFunctionStub - Return a stub for the function at the -/// specified address, created lazily on demand. -void *JITResolver::getExternalFunctionStub(void *FnAddr) { - // If we already have a stub for this function, recycle it. - void *&Stub = ExternalFnToStubMap[FnAddr]; - if (Stub) return Stub; - - TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); - JE.startGVStub(nullptr, SL.Size, SL.Alignment); - Stub = TheJIT->getJITInfo().emitFunctionStub(nullptr, FnAddr, JE); - JE.finishGVStub(); - - DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub - << "] for external function at '" << FnAddr << "'\n"); - return Stub; -} - -unsigned JITResolver::getGOTIndexForAddr(void* addr) { - unsigned idx = revGOTMap[addr]; - if (!idx) { - idx = ++nextGOTIndex; - revGOTMap[addr] = idx; - DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" - << addr << "]\n"); - } - return idx; -} - -/// JITCompilerFn - This function is called when a lazy compilation stub has -/// been entered. It looks up which function this stub corresponds to, compiles -/// it if necessary, then returns the resultant function pointer. -void *JITResolver::JITCompilerFn(void *Stub) { - JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); - assert(JR && "Unable to find the corresponding JITResolver to the call site"); - - Function* F = nullptr; - void* ActualPtr = nullptr; - - { - // Only lock for getting the Function. The call getPointerToFunction made - // in this function might trigger function materializing, which requires - // JIT lock to be unlocked. - MutexGuard locked(JR->TheJIT->lock); - - // The address given to us for the stub may not be exactly right, it might - // be a little bit after the stub. As such, use upper_bound to find it. - std::pair<void*, Function*> I = - JR->state.LookupFunctionFromCallSite(Stub); - F = I.second; - ActualPtr = I.first; - } - - // If we have already code generated the function, just return the address. - void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); - - if (!Result) { - // Otherwise we don't have it, do lazy compilation now. - - // If lazy compilation is disabled, emit a useful error message and abort. - if (!JR->TheJIT->isCompilingLazily()) { - report_fatal_error("LLVM JIT requested to do lazy compilation of" - " function '" - + F->getName() + "' when lazy compiles are disabled!"); - } - - DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() - << "' In stub ptr = " << Stub << " actual ptr = " - << ActualPtr << "\n"); - (void)ActualPtr; - - Result = JR->TheJIT->getPointerToFunction(F); - } - - // Reacquire the lock to update the GOT map. - MutexGuard locked(JR->TheJIT->lock); - - // We might like to remove the call site from the CallSiteToFunction map, but - // we can't do that! Multiple threads could be stuck, waiting to acquire the - // lock above. As soon as the 1st function finishes compiling the function, - // the next one will be released, and needs to be able to find the function it - // needs to call. - - // FIXME: We could rewrite all references to this stub if we knew them. - - // What we will do is set the compiled function address to map to the - // same GOT entry as the stub so that later clients may update the GOT - // if they see it still using the stub address. - // Note: this is done so the Resolver doesn't have to manage GOT memory - // Do this without allocating map space if the target isn't using a GOT - if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) - JR->revGOTMap[Result] = JR->revGOTMap[Stub]; - - return Result; -} - -//===----------------------------------------------------------------------===// -// JITEmitter code. -// - -static GlobalObject *getSimpleAliasee(Constant *C) { - C = C->stripPointerCasts(); - return dyn_cast<GlobalObject>(C); -} - -void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, - bool MayNeedFarStub) { - if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) - return TheJIT->getOrEmitGlobalVariable(GV); - - if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) { - // We can only handle simple cases. - if (GlobalValue *GV = getSimpleAliasee(GA->getAliasee())) - return TheJIT->getPointerToGlobal(GV); - return nullptr; - } - - // If we have already compiled the function, return a pointer to its body. - Function *F = cast<Function>(V); - - void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); - if (FnStub) { - // Return the function stub if it's already created. We do this first so - // that we're returning the same address for the function as any previous - // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be - // close enough to call. - return FnStub; - } - - // If we know the target can handle arbitrary-distance calls, try to - // return a direct pointer. - if (!MayNeedFarStub) { - // If we have code, go ahead and return that. - void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); - if (ResultPtr) return ResultPtr; - - // If this is an external function pointer, we can force the JIT to - // 'compile' it, which really just adds it to the map. - if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) - return TheJIT->getPointerToFunction(F); - } - - // Otherwise, we may need a to emit a stub, and, conservatively, we always do - // so. Note that it's possible to return null from getLazyFunctionStub in the - // case of a weak extern that fails to resolve. - return Resolver.getLazyFunctionStub(F); -} - -void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { - // Make sure GV is emitted first, and create a stub containing the fully - // resolved address. - void *GVAddress = getPointerToGlobal(V, Reference, false); - void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); - return StubAddr; -} - -void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { - if (DL.isUnknown()) return; - if (!BeforePrintingInsn) return; - - const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext(); - - if (DL.getScope(Context) != nullptr && PrevDL != DL) { - JITEvent_EmittedFunctionDetails::LineStart NextLine; - NextLine.Address = getCurrentPCValue(); - NextLine.Loc = DL; - EmissionDetails.LineStarts.push_back(NextLine); - } - - PrevDL = DL; -} - -static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, - const DataLayout *TD) { - const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); - if (Constants.empty()) return 0; - - unsigned Size = 0; - for (unsigned i = 0, e = Constants.size(); i != e; ++i) { - MachineConstantPoolEntry CPE = Constants[i]; - unsigned AlignMask = CPE.getAlignment() - 1; - Size = (Size + AlignMask) & ~AlignMask; - Type *Ty = CPE.getType(); - Size += TD->getTypeAllocSize(Ty); - } - return Size; -} - -void JITEmitter::startFunction(MachineFunction &F) { - DEBUG(dbgs() << "JIT: Starting CodeGen of Function " - << F.getName() << "\n"); - - uintptr_t ActualSize = 0; - // Set the memory writable, if it's not already - MemMgr->setMemoryWritable(); - - if (SizeEstimate > 0) { - // SizeEstimate will be non-zero on reallocation attempts. - ActualSize = SizeEstimate; - } - - BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), - ActualSize); - BufferEnd = BufferBegin+ActualSize; - EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; - - // Ensure the constant pool/jump table info is at least 4-byte aligned. - emitAlignment(16); - - emitConstantPool(F.getConstantPool()); - if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) - initJumpTableInfo(MJTI); - - // About to start emitting the machine code for the function. - emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); - TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); - EmittedFunctions[F.getFunction()].Code = CurBufferPtr; - - MBBLocations.clear(); - - EmissionDetails.MF = &F; - EmissionDetails.LineStarts.clear(); -} - -bool JITEmitter::finishFunction(MachineFunction &F) { - if (CurBufferPtr == BufferEnd) { - // We must call endFunctionBody before retrying, because - // deallocateMemForFunction requires it. - MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); - retryWithMoreMemory(F); - return true; - } - - if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) - emitJumpTableInfo(MJTI); - - // FnStart is the start of the text, not the start of the constant pool and - // other per-function data. - uint8_t *FnStart = - (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); - - // FnEnd is the end of the function's machine code. - uint8_t *FnEnd = CurBufferPtr; - - if (!Relocations.empty()) { - CurFn = F.getFunction(); - NumRelos += Relocations.size(); - - // Resolve the relocations to concrete pointers. - for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { - MachineRelocation &MR = Relocations[i]; - void *ResultPtr = nullptr; - if (!MR.letTargetResolve()) { - if (MR.isExternalSymbol()) { - ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), - false); - DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" - << ResultPtr << "]\n"); - - // If the target REALLY wants a stub for this function, emit it now. - if (MR.mayNeedFarStub()) { - ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); - } - } else if (MR.isGlobalValue()) { - ResultPtr = getPointerToGlobal(MR.getGlobalValue(), - BufferBegin+MR.getMachineCodeOffset(), - MR.mayNeedFarStub()); - } else if (MR.isIndirectSymbol()) { - ResultPtr = getPointerToGVIndirectSym( - MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); - } else if (MR.isBasicBlock()) { - ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); - } else if (MR.isConstantPoolIndex()) { - ResultPtr = - (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); - } else { - assert(MR.isJumpTableIndex()); - ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); - } - - MR.setResultPointer(ResultPtr); - } - - // if we are managing the GOT and the relocation wants an index, - // give it one - if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { - unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); - MR.setGOTIndex(idx); - if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { - DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] - << "\n"); - ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; - } - } - } - - CurFn = nullptr; - TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], - Relocations.size(), MemMgr->getGOTBase()); - } - - // Update the GOT entry for F to point to the new code. - if (MemMgr->isManagingGOT()) { - unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); - if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { - DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] - << "\n"); - ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; - } - } - - // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for - // global variables that were referenced in the relocations. - MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); - - if (CurBufferPtr == BufferEnd) { - retryWithMoreMemory(F); - return true; - } else { - // Now that we've succeeded in emitting the function, reset the - // SizeEstimate back down to zero. - SizeEstimate = 0; - } - - BufferBegin = CurBufferPtr = nullptr; - NumBytes += FnEnd-FnStart; - - // Invalidate the icache if necessary. - sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); - - TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, - EmissionDetails); - - // Reset the previous debug location. - PrevDL = DebugLoc(); - - DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart - << "] Function: " << F.getName() - << ": " << (FnEnd-FnStart) << " bytes of text, " - << Relocations.size() << " relocations\n"); - - Relocations.clear(); - ConstPoolAddresses.clear(); - - // Mark code region readable and executable if it's not so already. - MemMgr->setMemoryExecutable(); - - DEBUG({ - dbgs() << "JIT: Binary code:\n"; - uint8_t* q = FnStart; - for (int i = 0; q < FnEnd; q += 4, ++i) { - if (i == 4) - i = 0; - if (i == 0) - dbgs() << "JIT: " << (long)(q - FnStart) << ": "; - bool Done = false; - for (int j = 3; j >= 0; --j) { - if (q + j >= FnEnd) - Done = true; - else - dbgs() << (unsigned short)q[j]; - } - if (Done) - break; - dbgs() << ' '; - if (i == 3) - dbgs() << '\n'; - } - dbgs()<< '\n'; - }); - - if (MMI) - MMI->EndFunction(); - - return false; -} - -void JITEmitter::retryWithMoreMemory(MachineFunction &F) { - DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); - Relocations.clear(); // Clear the old relocations or we'll reapply them. - ConstPoolAddresses.clear(); - ++NumRetries; - deallocateMemForFunction(F.getFunction()); - // Try again with at least twice as much free space. - SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); - - for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ - if (MBB->hasAddressTaken()) - TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); - } -} - -/// deallocateMemForFunction - Deallocate all memory for the specified -/// function body. Also drop any references the function has to stubs. -/// May be called while the Function is being destroyed inside ~Value(). -void JITEmitter::deallocateMemForFunction(const Function *F) { - ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator - Emitted = EmittedFunctions.find(F); - if (Emitted != EmittedFunctions.end()) { - MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); - TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); - - EmittedFunctions.erase(Emitted); - } -} - - -void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { - if (BufferBegin) - return JITCodeEmitter::allocateSpace(Size, Alignment); - - // create a new memory block if there is no active one. - // care must be taken so that BufferBegin is invalidated when a - // block is trimmed - BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); - BufferEnd = BufferBegin+Size; - return CurBufferPtr; -} - -void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { - // Delegate this call through the memory manager. - return MemMgr->allocateGlobal(Size, Alignment); -} - -void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { - if (TheJIT->getJITInfo().hasCustomConstantPool()) - return; - - const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); - if (Constants.empty()) return; - - unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout()); - unsigned Align = MCP->getConstantPoolAlignment(); - ConstantPoolBase = allocateSpace(Size, Align); - ConstantPool = MCP; - - if (!ConstantPoolBase) return; // Buffer overflow. - - DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase - << "] (size: " << Size << ", alignment: " << Align << ")\n"); - - // Initialize the memory for all of the constant pool entries. - unsigned Offset = 0; - for (unsigned i = 0, e = Constants.size(); i != e; ++i) { - MachineConstantPoolEntry CPE = Constants[i]; - unsigned AlignMask = CPE.getAlignment() - 1; - Offset = (Offset + AlignMask) & ~AlignMask; - - uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; - ConstPoolAddresses.push_back(CAddr); - if (CPE.isMachineConstantPoolEntry()) { - // FIXME: add support to lower machine constant pool values into bytes! - report_fatal_error("Initialize memory with machine specific constant pool" - "entry has not been implemented!"); - } - TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); - DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; - dbgs().write_hex(CAddr) << "]\n"); - - Type *Ty = CPE.Val.ConstVal->getType(); - Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty); - } -} - -void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { - if (TheJIT->getJITInfo().hasCustomJumpTables()) - return; - if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) - return; - - const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); - if (JT.empty()) return; - - unsigned NumEntries = 0; - for (unsigned i = 0, e = JT.size(); i != e; ++i) - NumEntries += JT[i].MBBs.size(); - - unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout()); - - // Just allocate space for all the jump tables now. We will fix up the actual - // MBB entries in the tables after we emit the code for each block, since then - // we will know the final locations of the MBBs in memory. - JumpTable = MJTI; - JumpTableBase = allocateSpace(NumEntries * EntrySize, - MJTI->getEntryAlignment(*TheJIT->getDataLayout())); -} - -void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { - if (TheJIT->getJITInfo().hasCustomJumpTables()) - return; - - const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); - if (JT.empty() || !JumpTableBase) return; - - - switch (MJTI->getEntryKind()) { - case MachineJumpTableInfo::EK_Inline: - return; - case MachineJumpTableInfo::EK_BlockAddress: { - // EK_BlockAddress - Each entry is a plain address of block, e.g.: - // .word LBB123 - assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) && - "Cross JIT'ing?"); - - // For each jump table, map each target in the jump table to the address of - // an emitted MachineBasicBlock. - intptr_t *SlotPtr = (intptr_t*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; - // Store the address of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) - *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); - } - break; - } - - case MachineJumpTableInfo::EK_Custom32: - case MachineJumpTableInfo::EK_GPRel32BlockAddress: - case MachineJumpTableInfo::EK_LabelDifference32: { - assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?"); - // For each jump table, place the offset from the beginning of the table - // to the target address. - int *SlotPtr = (int*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; - // Store the offset of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - uintptr_t Base = (uintptr_t)SlotPtr; - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { - uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); - /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. - *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); - } - } - break; - } - case MachineJumpTableInfo::EK_GPRel64BlockAddress: - llvm_unreachable( - "JT Info emission not implemented for GPRel64BlockAddress yet."); - } -} - -void JITEmitter::startGVStub(const GlobalValue* GV, - unsigned StubSize, unsigned Alignment) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); - BufferEnd = BufferBegin+StubSize+1; -} - -void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - BufferBegin = CurBufferPtr = (uint8_t *)Buffer; - BufferEnd = BufferBegin+StubSize+1; -} - -void JITEmitter::finishGVStub() { - assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); - NumBytes += getCurrentPCOffset(); - BufferBegin = SavedBufferBegin; - BufferEnd = SavedBufferEnd; - CurBufferPtr = SavedCurBufferPtr; -} - -void *JITEmitter::allocIndirectGV(const GlobalValue *GV, - const uint8_t *Buffer, size_t Size, - unsigned Alignment) { - uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); - memcpy(IndGV, Buffer, Size); - return IndGV; -} - -// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry -// in the constant pool that was last emitted with the 'emitConstantPool' -// method. -// -uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { - assert(ConstantNum < ConstantPool->getConstants().size() && - "Invalid ConstantPoolIndex!"); - return ConstPoolAddresses[ConstantNum]; -} - -// getJumpTableEntryAddress - Return the address of the JumpTable with index -// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' -// -uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { - const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); - assert(Index < JT.size() && "Invalid jump table index!"); - - unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout()); - - unsigned Offset = 0; - for (unsigned i = 0; i < Index; ++i) - Offset += JT[i].MBBs.size(); - - Offset *= EntrySize; - - return (uintptr_t)((char *)JumpTableBase + Offset); -} - -void JITEmitter::EmittedFunctionConfig::onDelete( - JITEmitter *Emitter, const Function *F) { - Emitter->deallocateMemForFunction(F); -} -void JITEmitter::EmittedFunctionConfig::onRAUW( - JITEmitter *, const Function*, const Function*) { - llvm_unreachable("The JIT doesn't know how to handle a" - " RAUW on a value it has emitted."); -} - - -//===----------------------------------------------------------------------===// -// Public interface to this file -//===----------------------------------------------------------------------===// - -JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, - TargetMachine &tm) { - return new JITEmitter(jit, JMM, tm); -} - -// getPointerToFunctionOrStub - If the specified function has been -// code-gen'd, return a pointer to the function. If not, compile it, or use -// a stub to implement lazy compilation if available. -// -void *JIT::getPointerToFunctionOrStub(Function *F) { - // If we have already code generated the function, just return the address. - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; - - // Get a stub if the target supports it. - JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); - return JE->getJITResolver().getLazyFunctionStub(F); -} - -void JIT::updateFunctionStubUnlocked(Function *F) { - // Get the empty stub we generated earlier. - JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); - void *Stub = JE->getJITResolver().getLazyFunctionStub(F); - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr != Stub && "Function must have non-stub address to be updated."); - - // Tell the target jit info to rewrite the stub at the specified address, - // rather than creating a new one. - TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); - JE->startGVStub(Stub, layout.Size); - getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); - JE->finishGVStub(); -} - -/// freeMachineCodeForFunction - release machine code memory for given Function. -/// -void JIT::freeMachineCodeForFunction(Function *F) { - // Delete translation for this from the ExecutionEngine, so it will get - // retranslated next time it is used. - updateGlobalMapping(F, nullptr); - - // Free the actual memory for the function body and related stuff. - static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F); -} diff --git a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp deleted file mode 100644 index 584b93f81502..000000000000 --- a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp +++ /dev/null @@ -1,904 +0,0 @@ -//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the DefaultJITMemoryManager class. -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Config/config.h" -#include "llvm/IR/GlobalValue.h" -#include "llvm/Support/Allocator.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/raw_ostream.h" -#include <cassert> -#include <climits> -#include <cstring> -#include <vector> - -#if defined(__linux__) -#if defined(HAVE_SYS_STAT_H) -#include <sys/stat.h> -#endif -#include <fcntl.h> -#include <unistd.h> -#endif - -using namespace llvm; - -#define DEBUG_TYPE "jit" - -STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); - -JITMemoryManager::~JITMemoryManager() {} - -//===----------------------------------------------------------------------===// -// Memory Block Implementation. -//===----------------------------------------------------------------------===// - -namespace { - /// MemoryRangeHeader - For a range of memory, this is the header that we put - /// on the block of memory. It is carefully crafted to be one word of memory. - /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader - /// which starts with this. - struct FreeRangeHeader; - struct MemoryRangeHeader { - /// ThisAllocated - This is true if this block is currently allocated. If - /// not, this can be converted to a FreeRangeHeader. - unsigned ThisAllocated : 1; - - /// PrevAllocated - Keep track of whether the block immediately before us is - /// allocated. If not, the word immediately before this header is the size - /// of the previous block. - unsigned PrevAllocated : 1; - - /// BlockSize - This is the size in bytes of this memory block, - /// including this header. - uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); - - - /// getBlockAfter - Return the memory block immediately after this one. - /// - MemoryRangeHeader &getBlockAfter() const { - return *reinterpret_cast<MemoryRangeHeader *>( - reinterpret_cast<char*>( - const_cast<MemoryRangeHeader *>(this))+BlockSize); - } - - /// getFreeBlockBefore - If the block before this one is free, return it, - /// otherwise return null. - FreeRangeHeader *getFreeBlockBefore() const { - if (PrevAllocated) return nullptr; - intptr_t PrevSize = reinterpret_cast<intptr_t *>( - const_cast<MemoryRangeHeader *>(this))[-1]; - return reinterpret_cast<FreeRangeHeader *>( - reinterpret_cast<char*>( - const_cast<MemoryRangeHeader *>(this))-PrevSize); - } - - /// FreeBlock - Turn an allocated block into a free block, adjusting - /// bits in the object headers, and adding an end of region memory block. - FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); - - /// TrimAllocationToSize - If this allocated block is significantly larger - /// than NewSize, split it into two pieces (where the former is NewSize - /// bytes, including the header), and add the new block to the free list. - FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, - uint64_t NewSize); - }; - - /// FreeRangeHeader - For a memory block that isn't already allocated, this - /// keeps track of the current block and has a pointer to the next free block. - /// Free blocks are kept on a circularly linked list. - struct FreeRangeHeader : public MemoryRangeHeader { - FreeRangeHeader *Prev; - FreeRangeHeader *Next; - - /// getMinBlockSize - Get the minimum size for a memory block. Blocks - /// smaller than this size cannot be created. - static unsigned getMinBlockSize() { - return sizeof(FreeRangeHeader)+sizeof(intptr_t); - } - - /// SetEndOfBlockSizeMarker - The word at the end of every free block is - /// known to be the size of the free block. Set it for this block. - void SetEndOfBlockSizeMarker() { - void *EndOfBlock = (char*)this + BlockSize; - ((intptr_t *)EndOfBlock)[-1] = BlockSize; - } - - FreeRangeHeader *RemoveFromFreeList() { - assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); - Next->Prev = Prev; - return Prev->Next = Next; - } - - void AddToFreeList(FreeRangeHeader *FreeList) { - Next = FreeList; - Prev = FreeList->Prev; - Prev->Next = this; - Next->Prev = this; - } - - /// GrowBlock - The block after this block just got deallocated. Merge it - /// into the current block. - void GrowBlock(uintptr_t NewSize); - - /// AllocateBlock - Mark this entire block allocated, updating freelists - /// etc. This returns a pointer to the circular free-list. - FreeRangeHeader *AllocateBlock(); - }; -} - - -/// AllocateBlock - Mark this entire block allocated, updating freelists -/// etc. This returns a pointer to the circular free-list. -FreeRangeHeader *FreeRangeHeader::AllocateBlock() { - assert(!ThisAllocated && !getBlockAfter().PrevAllocated && - "Cannot allocate an allocated block!"); - // Mark this block allocated. - ThisAllocated = 1; - getBlockAfter().PrevAllocated = 1; - - // Remove it from the free list. - return RemoveFromFreeList(); -} - -/// FreeBlock - Turn an allocated block into a free block, adjusting -/// bits in the object headers, and adding an end of region memory block. -/// If possible, coalesce this block with neighboring blocks. Return the -/// FreeRangeHeader to allocate from. -FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { - MemoryRangeHeader *FollowingBlock = &getBlockAfter(); - assert(ThisAllocated && "This block is already free!"); - assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); - - FreeRangeHeader *FreeListToReturn = FreeList; - - // If the block after this one is free, merge it into this block. - if (!FollowingBlock->ThisAllocated) { - FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; - // "FreeList" always needs to be a valid free block. If we're about to - // coalesce with it, update our notion of what the free list is. - if (&FollowingFreeBlock == FreeList) { - FreeList = FollowingFreeBlock.Next; - FreeListToReturn = nullptr; - assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); - } - FollowingFreeBlock.RemoveFromFreeList(); - - // Include the following block into this one. - BlockSize += FollowingFreeBlock.BlockSize; - FollowingBlock = &FollowingFreeBlock.getBlockAfter(); - - // Tell the block after the block we are coalescing that this block is - // allocated. - FollowingBlock->PrevAllocated = 1; - } - - assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); - - if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { - PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); - return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; - } - - // Otherwise, mark this block free. - FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; - FollowingBlock->PrevAllocated = 0; - FreeBlock.ThisAllocated = 0; - - // Link this into the linked list of free blocks. - FreeBlock.AddToFreeList(FreeList); - - // Add a marker at the end of the block, indicating the size of this free - // block. - FreeBlock.SetEndOfBlockSizeMarker(); - return FreeListToReturn ? FreeListToReturn : &FreeBlock; -} - -/// GrowBlock - The block after this block just got deallocated. Merge it -/// into the current block. -void FreeRangeHeader::GrowBlock(uintptr_t NewSize) { - assert(NewSize > BlockSize && "Not growing block?"); - BlockSize = NewSize; - SetEndOfBlockSizeMarker(); - getBlockAfter().PrevAllocated = 0; -} - -/// TrimAllocationToSize - If this allocated block is significantly larger -/// than NewSize, split it into two pieces (where the former is NewSize -/// bytes, including the header), and add the new block to the free list. -FreeRangeHeader *MemoryRangeHeader:: -TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { - assert(ThisAllocated && getBlockAfter().PrevAllocated && - "Cannot deallocate part of an allocated block!"); - - // Don't allow blocks to be trimmed below minimum required size - NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); - - // Round up size for alignment of header. - unsigned HeaderAlign = __alignof(FreeRangeHeader); - NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); - - // Size is now the size of the block we will remove from the start of the - // current block. - assert(NewSize <= BlockSize && - "Allocating more space from this block than exists!"); - - // If splitting this block will cause the remainder to be too small, do not - // split the block. - if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) - return FreeList; - - // Otherwise, we splice the required number of bytes out of this block, form - // a new block immediately after it, then mark this block allocated. - MemoryRangeHeader &FormerNextBlock = getBlockAfter(); - - // Change the size of this block. - BlockSize = NewSize; - - // Get the new block we just sliced out and turn it into a free block. - FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); - NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; - NewNextBlock.ThisAllocated = 0; - NewNextBlock.PrevAllocated = 1; - NewNextBlock.SetEndOfBlockSizeMarker(); - FormerNextBlock.PrevAllocated = 0; - NewNextBlock.AddToFreeList(FreeList); - return &NewNextBlock; -} - -//===----------------------------------------------------------------------===// -// Memory Block Implementation. -//===----------------------------------------------------------------------===// - -namespace { - - class DefaultJITMemoryManager; - - class JITAllocator { - DefaultJITMemoryManager &JMM; - public: - JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } - void *Allocate(size_t Size, size_t /*Alignment*/); - void Deallocate(void *Slab, size_t Size); - }; - - /// DefaultJITMemoryManager - Manage memory for the JIT code generation. - /// This splits a large block of MAP_NORESERVE'd memory into two - /// sections, one for function stubs, one for the functions themselves. We - /// have to do this because we may need to emit a function stub while in the - /// middle of emitting a function, and we don't know how large the function we - /// are emitting is. - class DefaultJITMemoryManager : public JITMemoryManager { - public: - /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at - /// least this much unless more is requested. Currently, in 512k slabs. - static const size_t DefaultCodeSlabSize = 512 * 1024; - - /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably - /// 16 pages) unless we get an allocation above SizeThreshold. - static const size_t DefaultSlabSize = 64 * 1024; - - /// DefaultSizeThreshold - For any allocation larger than 16K (probably - /// 4 pages), we should allocate a separate slab to avoid wasted space at - /// the end of a normal slab. - static const size_t DefaultSizeThreshold = 16 * 1024; - - private: - // Whether to poison freed memory. - bool PoisonMemory; - - /// LastSlab - This points to the last slab allocated and is used as the - /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all - /// stubs, data, and code contiguously in memory. In general, however, this - /// is not possible because the NearBlock parameter is ignored on Windows - /// platforms and even on Unix it works on a best-effort pasis. - sys::MemoryBlock LastSlab; - - // Memory slabs allocated by the JIT. We refer to them as slabs so we don't - // confuse them with the blocks of memory described above. - std::vector<sys::MemoryBlock> CodeSlabs; - BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, - DefaultSizeThreshold> StubAllocator; - BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, - DefaultSizeThreshold> DataAllocator; - - // Circular list of free blocks. - FreeRangeHeader *FreeMemoryList; - - // When emitting code into a memory block, this is the block. - MemoryRangeHeader *CurBlock; - - uint8_t *GOTBase; // Target Specific reserved memory - public: - DefaultJITMemoryManager(); - ~DefaultJITMemoryManager(); - - /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the - /// last slab it allocated, so that subsequent allocations follow it. - sys::MemoryBlock allocateNewSlab(size_t size); - - /// getPointerToNamedFunction - This method returns the address of the - /// specified function by using the dlsym function call. - void *getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure = true) override; - - void AllocateGOT() override; - - // Testing methods. - bool CheckInvariants(std::string &ErrorStr) override; - size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; } - size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; } - size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; } - unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); } - unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); } - unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); } - - /// startFunctionBody - When a function starts, allocate a block of free - /// executable memory, returning a pointer to it and its actual size. - uint8_t *startFunctionBody(const Function *F, - uintptr_t &ActualSize) override { - - FreeRangeHeader* candidateBlock = FreeMemoryList; - FreeRangeHeader* head = FreeMemoryList; - FreeRangeHeader* iter = head->Next; - - uintptr_t largest = candidateBlock->BlockSize; - - // Search for the largest free block - while (iter != head) { - if (iter->BlockSize > largest) { - largest = iter->BlockSize; - candidateBlock = iter; - } - iter = iter->Next; - } - - largest = largest - sizeof(MemoryRangeHeader); - - // If this block isn't big enough for the allocation desired, allocate - // another block of memory and add it to the free list. - if (largest < ActualSize || - largest <= FreeRangeHeader::getMinBlockSize()) { - DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); - candidateBlock = allocateNewCodeSlab((size_t)ActualSize); - } - - // Select this candidate block for allocation - CurBlock = candidateBlock; - - // Allocate the entire memory block. - FreeMemoryList = candidateBlock->AllocateBlock(); - ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); - return (uint8_t *)(CurBlock + 1); - } - - /// allocateNewCodeSlab - Helper method to allocate a new slab of code - /// memory from the OS and add it to the free list. Returns the new - /// FreeRangeHeader at the base of the slab. - FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { - // If the user needs at least MinSize free memory, then we account for - // two MemoryRangeHeaders: the one in the user's block, and the one at the - // end of the slab. - size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); - size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); - sys::MemoryBlock B = allocateNewSlab(SlabSize); - CodeSlabs.push_back(B); - char *MemBase = (char*)(B.base()); - - // Put a tiny allocated block at the end of the memory chunk, so when - // FreeBlock calls getBlockAfter it doesn't fall off the end. - MemoryRangeHeader *EndBlock = - (MemoryRangeHeader*)(MemBase + B.size()) - 1; - EndBlock->ThisAllocated = 1; - EndBlock->PrevAllocated = 0; - EndBlock->BlockSize = sizeof(MemoryRangeHeader); - - // Start out with a vast new block of free memory. - FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; - NewBlock->ThisAllocated = 0; - // Make sure getFreeBlockBefore doesn't look into unmapped memory. - NewBlock->PrevAllocated = 1; - NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; - NewBlock->SetEndOfBlockSizeMarker(); - NewBlock->AddToFreeList(FreeMemoryList); - - assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && - "The block was too small!"); - return NewBlock; - } - - /// endFunctionBody - The function F is now allocated, and takes the memory - /// in the range [FunctionStart,FunctionEnd). - void endFunctionBody(const Function *F, uint8_t *FunctionStart, - uint8_t *FunctionEnd) override { - assert(FunctionEnd > FunctionStart); - assert(FunctionStart == (uint8_t *)(CurBlock+1) && - "Mismatched function start/end!"); - - uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; - - // Release the memory at the end of this block that isn't needed. - FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); - } - - /// allocateSpace - Allocate a memory block of the given size. This method - /// cannot be called between calls to startFunctionBody and endFunctionBody. - uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override { - CurBlock = FreeMemoryList; - FreeMemoryList = FreeMemoryList->AllocateBlock(); - - uint8_t *result = (uint8_t *)(CurBlock + 1); - - if (Alignment == 0) Alignment = 1; - result = (uint8_t*)(((intptr_t)result+Alignment-1) & - ~(intptr_t)(Alignment-1)); - - uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; - FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); - - return result; - } - - /// allocateStub - Allocate memory for a function stub. - uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, - unsigned Alignment) override { - return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); - } - - /// allocateGlobal - Allocate memory for a global. - uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override { - return (uint8_t*)DataAllocator.Allocate(Size, Alignment); - } - - /// allocateCodeSection - Allocate memory for a code section. - uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - // Grow the required block size to account for the block header - Size += sizeof(*CurBlock); - - // Alignment handling. - if (!Alignment) - Alignment = 16; - Size += Alignment - 1; - - FreeRangeHeader* candidateBlock = FreeMemoryList; - FreeRangeHeader* head = FreeMemoryList; - FreeRangeHeader* iter = head->Next; - - uintptr_t largest = candidateBlock->BlockSize; - - // Search for the largest free block. - while (iter != head) { - if (iter->BlockSize > largest) { - largest = iter->BlockSize; - candidateBlock = iter; - } - iter = iter->Next; - } - - largest = largest - sizeof(MemoryRangeHeader); - - // If this block isn't big enough for the allocation desired, allocate - // another block of memory and add it to the free list. - if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) { - DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); - candidateBlock = allocateNewCodeSlab((size_t)Size); - } - - // Select this candidate block for allocation - CurBlock = candidateBlock; - - // Allocate the entire memory block. - FreeMemoryList = candidateBlock->AllocateBlock(); - // Release the memory at the end of this block that isn't needed. - FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size); - uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock); - return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment); - } - - /// allocateDataSection - Allocate memory for a data section. - uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool IsReadOnly) override { - return (uint8_t*)DataAllocator.Allocate(Size, Alignment); - } - - bool finalizeMemory(std::string *ErrMsg) override { - return false; - } - - uint8_t *getGOTBase() const override { - return GOTBase; - } - - void deallocateBlock(void *Block) { - // Find the block that is allocated for this function. - MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1; - assert(MemRange->ThisAllocated && "Block isn't allocated!"); - - // Fill the buffer with garbage! - if (PoisonMemory) { - memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); - } - - // Free the memory. - FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); - } - - /// deallocateFunctionBody - Deallocate all memory for the specified - /// function body. - void deallocateFunctionBody(void *Body) override { - if (Body) deallocateBlock(Body); - } - - /// setMemoryWritable - When code generation is in progress, - /// the code pages may need permissions changed. - void setMemoryWritable() override { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::setWritable(CodeSlabs[i]); - } - /// setMemoryExecutable - When code generation is done and we're ready to - /// start execution, the code pages may need permissions changed. - void setMemoryExecutable() override { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::setExecutable(CodeSlabs[i]); - } - - /// setPoisonMemory - Controls whether we write garbage over freed memory. - /// - void setPoisonMemory(bool poison) override { - PoisonMemory = poison; - } - }; -} - -void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) { - sys::MemoryBlock B = JMM.allocateNewSlab(Size); - return B.base(); -} - -void JITAllocator::Deallocate(void *Slab, size_t Size) { - sys::MemoryBlock B(Slab, Size); - sys::Memory::ReleaseRWX(B); -} - -DefaultJITMemoryManager::DefaultJITMemoryManager() - : -#ifdef NDEBUG - PoisonMemory(false), -#else - PoisonMemory(true), -#endif - LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) { - - // Allocate space for code. - sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); - CodeSlabs.push_back(MemBlock); - uint8_t *MemBase = (uint8_t*)MemBlock.base(); - - // We set up the memory chunk with 4 mem regions, like this: - // [ START - // [ Free #0 ] -> Large space to allocate functions from. - // [ Allocated #1 ] -> Tiny space to separate regions. - // [ Free #2 ] -> Tiny space so there is always at least 1 free block. - // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. - // END ] - // - // The last three blocks are never deallocated or touched. - - // Add MemoryRangeHeader to the end of the memory region, indicating that - // the space after the block of memory is allocated. This is block #3. - MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; - Mem3->ThisAllocated = 1; - Mem3->PrevAllocated = 0; - Mem3->BlockSize = sizeof(MemoryRangeHeader); - - /// Add a tiny free region so that the free list always has one entry. - FreeRangeHeader *Mem2 = - (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); - Mem2->ThisAllocated = 0; - Mem2->PrevAllocated = 1; - Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); - Mem2->SetEndOfBlockSizeMarker(); - Mem2->Prev = Mem2; // Mem2 *is* the free list for now. - Mem2->Next = Mem2; - - /// Add a tiny allocated region so that Mem2 is never coalesced away. - MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; - Mem1->ThisAllocated = 1; - Mem1->PrevAllocated = 0; - Mem1->BlockSize = sizeof(MemoryRangeHeader); - - // Add a FreeRangeHeader to the start of the function body region, indicating - // that the space is free. Mark the previous block allocated so we never look - // at it. - FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; - Mem0->ThisAllocated = 0; - Mem0->PrevAllocated = 1; - Mem0->BlockSize = (char*)Mem1-(char*)Mem0; - Mem0->SetEndOfBlockSizeMarker(); - Mem0->AddToFreeList(Mem2); - - // Start out with the freelist pointing to Mem0. - FreeMemoryList = Mem0; - - GOTBase = nullptr; -} - -void DefaultJITMemoryManager::AllocateGOT() { - assert(!GOTBase && "Cannot allocate the got multiple times"); - GOTBase = new uint8_t[sizeof(void*) * 8192]; - HasGOT = true; -} - -DefaultJITMemoryManager::~DefaultJITMemoryManager() { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::ReleaseRWX(CodeSlabs[i]); - - delete[] GOTBase; -} - -sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { - // Allocate a new block close to the last one. - std::string ErrMsg; - sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr; - sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); - if (!B.base()) { - report_fatal_error("Allocation failed when allocating new memory in the" - " JIT\n" + Twine(ErrMsg)); - } - LastSlab = B; - ++NumSlabs; - // Initialize the slab to garbage when debugging. - if (PoisonMemory) { - memset(B.base(), 0xCD, B.size()); - } - return B; -} - -/// CheckInvariants - For testing only. Return "" if all internal invariants -/// are preserved, and a helpful error message otherwise. For free and -/// allocated blocks, make sure that adding BlockSize gives a valid block. -/// For free blocks, make sure they're in the free list and that their end of -/// block size marker is correct. This function should return an error before -/// accessing bad memory. This function is defined here instead of in -/// JITMemoryManagerTest.cpp so that we don't have to expose all of the -/// implementation details of DefaultJITMemoryManager. -bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { - raw_string_ostream Err(ErrorStr); - - // Construct a the set of FreeRangeHeader pointers so we can query it - // efficiently. - llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; - FreeRangeHeader* FreeHead = FreeMemoryList; - FreeRangeHeader* FreeRange = FreeHead; - - do { - // Check that the free range pointer is in the blocks we've allocated. - bool Found = false; - for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), - E = CodeSlabs.end(); I != E && !Found; ++I) { - char *Start = (char*)I->base(); - char *End = Start + I->size(); - Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); - } - if (!Found) { - Err << "Corrupt free list; points to " << FreeRange; - return false; - } - - if (FreeRange->Next->Prev != FreeRange) { - Err << "Next and Prev pointers do not match."; - return false; - } - - // Otherwise, add it to the set. - FreeHdrSet.insert(FreeRange); - FreeRange = FreeRange->Next; - } while (FreeRange != FreeHead); - - // Go over each block, and look at each MemoryRangeHeader. - for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), - E = CodeSlabs.end(); I != E; ++I) { - char *Start = (char*)I->base(); - char *End = Start + I->size(); - - // Check each memory range. - for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr; - Start <= (char*)Hdr && (char*)Hdr < End; - Hdr = &Hdr->getBlockAfter()) { - if (Hdr->ThisAllocated == 0) { - // Check that this range is in the free list. - if (!FreeHdrSet.count(Hdr)) { - Err << "Found free header at " << Hdr << " that is not in free list."; - return false; - } - - // Now make sure the size marker at the end of the block is correct. - uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; - if (!(Start <= (char*)Marker && (char*)Marker < End)) { - Err << "Block size in header points out of current MemoryBlock."; - return false; - } - if (Hdr->BlockSize != *Marker) { - Err << "End of block size marker (" << *Marker << ") " - << "and BlockSize (" << Hdr->BlockSize << ") don't match."; - return false; - } - } - - if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { - Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " - << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; - return false; - } else if (!LastHdr && !Hdr->PrevAllocated) { - Err << "The first header should have PrevAllocated true."; - return false; - } - - // Remember the last header. - LastHdr = Hdr; - } - } - - // All invariants are preserved. - return true; -} - -//===----------------------------------------------------------------------===// -// getPointerToNamedFunction() implementation. -//===----------------------------------------------------------------------===// - -// AtExitHandlers - List of functions to call when the program exits, -// registered with the atexit() library function. -static std::vector<void (*)()> AtExitHandlers; - -/// runAtExitHandlers - Run any functions registered by the program's -/// calls to atexit(3), which we intercept and store in -/// AtExitHandlers. -/// -static void runAtExitHandlers() { - while (!AtExitHandlers.empty()) { - void (*Fn)() = AtExitHandlers.back(); - AtExitHandlers.pop_back(); - Fn(); - } -} - -//===----------------------------------------------------------------------===// -// Function stubs that are invoked instead of certain library calls -// -// Force the following functions to be linked in to anything that uses the -// JIT. This is a hack designed to work around the all-too-clever Glibc -// strategy of making these functions work differently when inlined vs. when -// not inlined, and hiding their real definitions in a separate archive file -// that the dynamic linker can't see. For more info, search for -// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274. -#if defined(__linux__) && defined(__GLIBC__) -/* stat functions are redirecting to __xstat with a version number. On x86-64 - * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat' - * available as an exported symbol, so we have to add it explicitly. - */ -namespace { -class StatSymbols { -public: - StatSymbols() { - sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat); - sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat); - sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat); - sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64); - sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64); - sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64); - sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64); - sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64); - sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64); - sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit); - sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod); - } -}; -} -static StatSymbols initStatSymbols; -#endif // __linux__ - -// jit_exit - Used to intercept the "exit" library call. -static void jit_exit(int Status) { - runAtExitHandlers(); // Run atexit handlers... - exit(Status); -} - -// jit_atexit - Used to intercept the "atexit" library call. -static int jit_atexit(void (*Fn)()) { - AtExitHandlers.push_back(Fn); // Take note of atexit handler... - return 0; // Always successful -} - -static int jit_noop() { - return 0; -} - -//===----------------------------------------------------------------------===// -// -/// getPointerToNamedFunction - This method returns the address of the specified -/// function by using the dynamic loader interface. As such it is only useful -/// for resolving library symbols, not code generated symbols. -/// -void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure) { - // Check to see if this is one of the functions we want to intercept. Note, - // we cast to intptr_t here to silence a -pedantic warning that complains - // about casting a function pointer to a normal pointer. - if (Name == "exit") return (void*)(intptr_t)&jit_exit; - if (Name == "atexit") return (void*)(intptr_t)&jit_atexit; - - // We should not invoke parent's ctors/dtors from generated main()! - // On Mingw and Cygwin, the symbol __main is resolved to - // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors - // (and register wrong callee's dtors with atexit(3)). - // We expect ExecutionEngine::runStaticConstructorsDestructors() - // is called before ExecutionEngine::runFunctionAsMain() is called. - if (Name == "__main") return (void*)(intptr_t)&jit_noop; - - const char *NameStr = Name.c_str(); - // If this is an asm specifier, skip the sentinal. - if (NameStr[0] == 1) ++NameStr; - - // If it's an external function, look it up in the process image... - void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); - if (Ptr) return Ptr; - - // If it wasn't found and if it starts with an underscore ('_') character, - // try again without the underscore. - if (NameStr[0] == '_') { - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); - if (Ptr) return Ptr; - } - - // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These - // are references to hidden visibility symbols that dlsym cannot resolve. - // If we have one of these, strip off $LDBLStub and try again. -#if defined(__APPLE__) && defined(__ppc__) - if (Name.size() > 9 && Name[Name.size()-9] == '$' && - memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) { - // First try turning $LDBLStub into $LDBL128. If that fails, strip it off. - // This mirrors logic in libSystemStubs.a. - std::string Prefix = std::string(Name.begin(), Name.end()-9); - if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false)) - return Ptr; - if (void *Ptr = getPointerToNamedFunction(Prefix, false)) - return Ptr; - } -#endif - - if (AbortOnFailure) { - report_fatal_error("Program used external function '"+Name+ - "' which could not be resolved!"); - } - return nullptr; -} - - - -JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { - return new DefaultJITMemoryManager(); -} - -const size_t DefaultJITMemoryManager::DefaultCodeSlabSize; -const size_t DefaultJITMemoryManager::DefaultSlabSize; -const size_t DefaultJITMemoryManager::DefaultSizeThreshold; diff --git a/lib/ExecutionEngine/JIT/LLVMBuild.txt b/lib/ExecutionEngine/JIT/LLVMBuild.txt deleted file mode 100644 index dd22f1b464a7..000000000000 --- a/lib/ExecutionEngine/JIT/LLVMBuild.txt +++ /dev/null @@ -1,22 +0,0 @@ -;===- ./lib/ExecutionEngine/JIT/LLVMBuild.txt ------------------*- Conf -*--===; -; -; The LLVM Compiler Infrastructure -; -; This file is distributed under the University of Illinois Open Source -; License. See LICENSE.TXT for details. -; -;===------------------------------------------------------------------------===; -; -; This is an LLVMBuild description file for the components in this subdirectory. -; -; For more information on the LLVMBuild system, please see: -; -; http://llvm.org/docs/LLVMBuild.html -; -;===------------------------------------------------------------------------===; - -[component_0] -type = Library -name = JIT -parent = ExecutionEngine -required_libraries = CodeGen Core ExecutionEngine Support diff --git a/lib/ExecutionEngine/JIT/Makefile b/lib/ExecutionEngine/JIT/Makefile deleted file mode 100644 index aafa3d9d420f..000000000000 --- a/lib/ExecutionEngine/JIT/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -##===- lib/ExecutionEngine/JIT/Makefile --------------------*- Makefile -*-===## -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -##===----------------------------------------------------------------------===## - -LEVEL = ../../.. -LIBRARYNAME = LLVMJIT - -# Get the $(ARCH) setting -include $(LEVEL)/Makefile.config - -# Enable the X86 JIT if compiling on X86 -ifeq ($(ARCH), x86) - ENABLE_X86_JIT = 1 -endif - -# This flag can also be used on the command line to force inclusion -# of the X86 JIT on non-X86 hosts -ifdef ENABLE_X86_JIT - CPPFLAGS += -DENABLE_X86_JIT -endif - -# Enable the Sparc JIT if compiling on Sparc -ifeq ($(ARCH), Sparc) - ENABLE_SPARC_JIT = 1 -endif - -# This flag can also be used on the command line to force inclusion -# of the Sparc JIT on non-Sparc hosts -ifdef ENABLE_SPARC_JIT - CPPFLAGS += -DENABLE_SPARC_JIT -endif - -include $(LEVEL)/Makefile.common diff --git a/lib/ExecutionEngine/LLVMBuild.txt b/lib/ExecutionEngine/LLVMBuild.txt index 6dc75af2ec96..7d041cef843e 100644 --- a/lib/ExecutionEngine/LLVMBuild.txt +++ b/lib/ExecutionEngine/LLVMBuild.txt @@ -16,10 +16,10 @@ ;===------------------------------------------------------------------------===; [common] -subdirectories = Interpreter JIT MCJIT RuntimeDyld IntelJITEvents OProfileJIT +subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT [component_0] type = Library name = ExecutionEngine parent = Libraries -required_libraries = Core MC Support +required_libraries = Core MC Object Support diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp index e9ba96a6496f..f2d53f5326d1 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -10,10 +10,7 @@ #include "MCJIT.h" #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" #include "llvm/ExecutionEngine/MCJIT.h" -#include "llvm/ExecutionEngine/ObjectBuffer.h" -#include "llvm/ExecutionEngine/ObjectImage.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" @@ -22,15 +19,19 @@ #include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/Object/Archive.h" +#include "llvm/Object/ObjectFile.h" #include "llvm/PassManager.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/MutexGuard.h" #include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; +void ObjectCache::anchor() {} + namespace { static struct RegisterJIT { @@ -42,31 +43,26 @@ static struct RegisterJIT { extern "C" void LLVMLinkInMCJIT() { } -ExecutionEngine *MCJIT::createJIT(Module *M, +ExecutionEngine *MCJIT::createJIT(std::unique_ptr<Module> M, std::string *ErrorStr, - RTDyldMemoryManager *MemMgr, - bool GVsWithCode, - TargetMachine *TM) { + std::unique_ptr<RTDyldMemoryManager> MemMgr, + std::unique_ptr<TargetMachine> TM) { // Try to register the program as a source of symbols to resolve against. // // FIXME: Don't do this here. sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr); - return new MCJIT(M, TM, MemMgr ? MemMgr : new SectionMemoryManager(), - GVsWithCode); -} - -MCJIT::MCJIT(Module *m, TargetMachine *tm, RTDyldMemoryManager *MM, - bool AllocateGVsWithCode) - : ExecutionEngine(m), TM(tm), Ctx(nullptr), MemMgr(this, MM), Dyld(&MemMgr), - ObjCache(nullptr) { + std::unique_ptr<RTDyldMemoryManager> MM = std::move(MemMgr); + if (!MM) + MM = std::unique_ptr<SectionMemoryManager>(new SectionMemoryManager()); - OwnedModules.addModule(m); - setDataLayout(TM->getDataLayout()); + return new MCJIT(std::move(M), std::move(TM), std::move(MM)); } -MCJIT::~MCJIT() { - MutexGuard locked(lock); +MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm, + std::unique_ptr<RTDyldMemoryManager> MM) + : ExecutionEngine(std::move(M)), TM(std::move(tm)), Ctx(nullptr), + MemMgr(this, std::move(MM)), Dyld(&MemMgr), ObjCache(nullptr) { // FIXME: We are managing our modules, so we do not want the base class // ExecutionEngine to manage them as well. To avoid double destruction // of the first (and only) module added in ExecutionEngine constructor @@ -77,33 +73,29 @@ MCJIT::~MCJIT() { // If so, additional functions: addModule, removeModule, FindFunctionNamed, // runStaticConstructorsDestructors could be moved back to EE as well. // + std::unique_ptr<Module> First = std::move(Modules[0]); Modules.clear(); + + OwnedModules.addModule(std::move(First)); + setDataLayout(TM->getSubtargetImpl()->getDataLayout()); + RegisterJITEventListener(JITEventListener::createGDBRegistrationListener()); +} + +MCJIT::~MCJIT() { + MutexGuard locked(lock); + Dyld.deregisterEHFrames(); - LoadedObjectList::iterator it, end; - for (it = LoadedObjects.begin(), end = LoadedObjects.end(); it != end; ++it) { - ObjectImage *Obj = *it; - if (Obj) { + for (auto &Obj : LoadedObjects) + if (Obj) NotifyFreeingObject(*Obj); - delete Obj; - } - } - LoadedObjects.clear(); - - SmallVector<object::Archive *, 2>::iterator ArIt, ArEnd; - for (ArIt = Archives.begin(), ArEnd = Archives.end(); ArIt != ArEnd; ++ArIt) { - object::Archive *A = *ArIt; - delete A; - } Archives.clear(); - - delete TM; } -void MCJIT::addModule(Module *M) { +void MCJIT::addModule(std::unique_ptr<Module> M) { MutexGuard locked(lock); - OwnedModules.addModule(M); + OwnedModules.addModule(std::move(M)); } bool MCJIT::removeModule(Module *M) { @@ -111,29 +103,34 @@ bool MCJIT::removeModule(Module *M) { return OwnedModules.removeModule(M); } - - void MCJIT::addObjectFile(std::unique_ptr<object::ObjectFile> Obj) { - ObjectImage *LoadedObject = Dyld.loadObject(std::move(Obj)); - if (!LoadedObject || Dyld.hasError()) + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> L = Dyld.loadObject(*Obj); + if (Dyld.hasError()) report_fatal_error(Dyld.getErrorString()); - LoadedObjects.push_back(LoadedObject); + NotifyObjectEmitted(*Obj, *L); - NotifyObjectEmitted(*LoadedObject); + LoadedObjects.push_back(std::move(Obj)); } -void MCJIT::addArchive(object::Archive *A) { - Archives.push_back(A); +void MCJIT::addObjectFile(object::OwningBinary<object::ObjectFile> Obj) { + std::unique_ptr<object::ObjectFile> ObjFile; + std::unique_ptr<MemoryBuffer> MemBuf; + std::tie(ObjFile, MemBuf) = Obj.takeBinary(); + addObjectFile(std::move(ObjFile)); + Buffers.push_back(std::move(MemBuf)); } +void MCJIT::addArchive(object::OwningBinary<object::Archive> A) { + Archives.push_back(std::move(A)); +} void MCJIT::setObjectCache(ObjectCache* NewCache) { MutexGuard locked(lock); ObjCache = NewCache; } -ObjectBufferStream* MCJIT::emitObject(Module *M) { +std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) { MutexGuard locked(lock); // This must be a module which has already been added but not loaded to this @@ -142,34 +139,36 @@ ObjectBufferStream* MCJIT::emitObject(Module *M) { PassManager PM; - M->setDataLayout(TM->getDataLayout()); - PM.add(new DataLayoutPass(M)); + M->setDataLayout(TM->getSubtargetImpl()->getDataLayout()); + PM.add(new DataLayoutPass()); // The RuntimeDyld will take ownership of this shortly - std::unique_ptr<ObjectBufferStream> CompiledObject(new ObjectBufferStream()); + SmallVector<char, 4096> ObjBufferSV; + raw_svector_ostream ObjStream(ObjBufferSV); // Turn the machine code intermediate representation into bytes in memory // that may be executed. - if (TM->addPassesToEmitMC(PM, Ctx, CompiledObject->getOStream(), - !getVerifyModules())) { + if (TM->addPassesToEmitMC(PM, Ctx, ObjStream, !getVerifyModules())) report_fatal_error("Target does not support MC emission!"); - } // Initialize passes. PM.run(*M); // Flush the output buffer to get the generated code into memory - CompiledObject->flush(); + ObjStream.flush(); + + std::unique_ptr<MemoryBuffer> CompiledObjBuffer( + new ObjectMemoryBuffer(std::move(ObjBufferSV))); // If we have an object cache, tell it about the new object. // Note that we're using the compiled image, not the loaded image (as below). if (ObjCache) { // MemoryBuffer is a thin wrapper around the actual memory, so it's OK // to create a temporary object here and delete it after the call. - std::unique_ptr<MemoryBuffer> MB(CompiledObject->getMemBuffer()); - ObjCache->notifyObjectCompiled(M, MB.get()); + MemoryBufferRef MB = CompiledObjBuffer->getMemBufferRef(); + ObjCache->notifyObjectCompiled(M, MB); } - return CompiledObject.release(); + return CompiledObjBuffer; } void MCJIT::generateCodeForModule(Module *M) { @@ -184,31 +183,31 @@ void MCJIT::generateCodeForModule(Module *M) { if (OwnedModules.hasModuleBeenLoaded(M)) return; - std::unique_ptr<ObjectBuffer> ObjectToLoad; + std::unique_ptr<MemoryBuffer> ObjectToLoad; // Try to load the pre-compiled object from cache if possible - if (ObjCache) { - std::unique_ptr<MemoryBuffer> PreCompiledObject(ObjCache->getObject(M)); - if (PreCompiledObject.get()) - ObjectToLoad.reset(new ObjectBuffer(PreCompiledObject.release())); - } + if (ObjCache) + ObjectToLoad = ObjCache->getObject(M); // If the cache did not contain a suitable object, compile the object if (!ObjectToLoad) { - ObjectToLoad.reset(emitObject(M)); - assert(ObjectToLoad.get() && "Compilation did not produce an object."); + ObjectToLoad = emitObject(M); + assert(ObjectToLoad && "Compilation did not produce an object."); } // Load the object into the dynamic linker. // MCJIT now owns the ObjectImage pointer (via its LoadedObjects list). - ObjectImage *LoadedObject = Dyld.loadObject(ObjectToLoad.release()); - LoadedObjects.push_back(LoadedObject); - if (!LoadedObject) + ErrorOr<std::unique_ptr<object::ObjectFile>> LoadedObject = + object::ObjectFile::createObjectFile(ObjectToLoad->getMemBufferRef()); + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> L = + Dyld.loadObject(*LoadedObject.get()); + + if (Dyld.hasError()) report_fatal_error(Dyld.getErrorString()); - // FIXME: Make this optional, maybe even move it to a JIT event listener - LoadedObject->registerWithDebugger(); + NotifyObjectEmitted(*LoadedObject.get(), *L); - NotifyObjectEmitted(*LoadedObject); + Buffers.push_back(std::move(ObjectToLoad)); + LoadedObjects.push_back(std::move(*LoadedObject)); OwnedModules.markModuleAsLoaded(M); } @@ -232,12 +231,14 @@ void MCJIT::finalizeLoadedModules() { void MCJIT::finalizeObject() { MutexGuard locked(lock); - for (ModulePtrSet::iterator I = OwnedModules.begin_added(), - E = OwnedModules.end_added(); - I != E; ++I) { - Module *M = *I; + // Generate code for module is going to move objects out of the 'added' list, + // so we need to copy that out before using it: + SmallVector<Module*, 16> ModsToAdd; + for (auto M : OwnedModules.added()) + ModsToAdd.push_back(M); + + for (auto M : ModsToAdd) generateCodeForModule(M); - } finalizeLoadedModules(); } @@ -255,12 +256,8 @@ void MCJIT::finalizeModule(Module *M) { finalizeLoadedModules(); } -void *MCJIT::getPointerToBasicBlock(BasicBlock *BB) { - report_fatal_error("not yet implemented"); -} - uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) { - Mangler Mang(TM->getDataLayout()); + Mangler Mang(TM->getSubtargetImpl()->getDataLayout()); SmallString<128> FullName; Mang.getNameWithPrefix(FullName, Name); return Dyld.getSymbolLoadAddress(FullName); @@ -299,9 +296,8 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, if (Addr) return Addr; - SmallVector<object::Archive*, 2>::iterator I, E; - for (I = Archives.begin(), E = Archives.end(); I != E; ++I) { - object::Archive *A = *I; + for (object::OwningBinary<object::Archive> &OB : Archives) { + object::Archive *A = OB.getBinary(); // Look for our symbols in each Archive object::Archive::child_iterator ChildIt = A->findSym(Name); if (ChildIt != A->child_end()) { @@ -310,7 +306,7 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, ChildIt->getAsBinary(); if (ChildBinOrErr.getError()) continue; - std::unique_ptr<object::Binary> ChildBin = std::move(ChildBinOrErr.get()); + std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); if (ChildBin->isObject()) { std::unique_ptr<object::ObjectFile> OF( static_cast<object::ObjectFile *>(ChildBin.release())); @@ -326,13 +322,19 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, // If it hasn't already been generated, see if it's in one of our modules. Module *M = findModuleForSymbol(Name, CheckFunctionsOnly); - if (!M) - return 0; + if (M) { + generateCodeForModule(M); - generateCodeForModule(M); + // Check the RuntimeDyld table again, it should be there now. + return getExistingSymbolAddress(Name); + } + + // If a LazyFunctionCreator is installed, use it to get/create the function. + // FIXME: Should we instead have a LazySymbolCreator callback? + if (LazyFunctionCreator) + Addr = (uint64_t)LazyFunctionCreator(Name); - // Check the RuntimeDyld table again, it should be there now. - return getExistingSymbolAddress(Name); + return Addr; } uint64_t MCJIT::getGlobalValueAddress(const std::string &Name) { @@ -355,10 +357,14 @@ uint64_t MCJIT::getFunctionAddress(const std::string &Name) { void *MCJIT::getPointerToFunction(Function *F) { MutexGuard locked(lock); + Mangler Mang(TM->getSubtargetImpl()->getDataLayout()); + SmallString<128> Name; + TM->getNameWithPrefix(Name, F, Mang); + if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) { bool AbortOnFailure = !F->hasExternalWeakLinkage(); - void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure); - addGlobalMapping(F, Addr); + void *Addr = getPointerToNamedFunction(Name, AbortOnFailure); + updateGlobalMapping(F, Addr); return Addr; } @@ -368,32 +374,25 @@ void *MCJIT::getPointerToFunction(Function *F) { // Make sure the relevant module has been compiled and loaded. if (HasBeenAddedButNotLoaded) generateCodeForModule(M); - else if (!OwnedModules.hasModuleBeenLoaded(M)) + else if (!OwnedModules.hasModuleBeenLoaded(M)) { // If this function doesn't belong to one of our modules, we're done. + // FIXME: Asking for the pointer to a function that hasn't been registered, + // and isn't a declaration (which is handled above) should probably + // be an assertion. return nullptr; + } // FIXME: Should the Dyld be retaining module information? Probably not. // // This is the accessor for the target address, so make sure to check the // load address of the symbol, not the local address. - Mangler Mang(TM->getDataLayout()); - SmallString<128> Name; - TM->getNameWithPrefix(Name, F, Mang); return (void*)Dyld.getSymbolLoadAddress(Name); } -void *MCJIT::recompileAndRelinkFunction(Function *F) { - report_fatal_error("not yet implemented"); -} - -void MCJIT::freeMachineCodeForFunction(Function *F) { - report_fatal_error("not yet implemented"); -} - void MCJIT::runStaticConstructorsDestructorsInModulePtrSet( bool isDtors, ModulePtrSet::iterator I, ModulePtrSet::iterator E) { for (; I != E; ++I) { - ExecutionEngine::runStaticConstructorsDestructors(*I, isDtors); + ExecutionEngine::runStaticConstructorsDestructors(**I, isDtors); } } @@ -529,8 +528,7 @@ GenericValue MCJIT::runFunction(Function *F, llvm_unreachable("Full-featured argument passing not supported yet!"); } -void *MCJIT::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure) { +void *MCJIT::getPointerToNamedFunction(StringRef Name, bool AbortOnFailure) { if (!isSymbolSearchingDisabled()) { void *ptr = MemMgr.getPointerToNamedFunction(Name, false); if (ptr) @@ -555,29 +553,31 @@ void MCJIT::RegisterJITEventListener(JITEventListener *L) { MutexGuard locked(lock); EventListeners.push_back(L); } + void MCJIT::UnregisterJITEventListener(JITEventListener *L) { if (!L) return; MutexGuard locked(lock); - SmallVector<JITEventListener*, 2>::reverse_iterator I= - std::find(EventListeners.rbegin(), EventListeners.rend(), L); + auto I = std::find(EventListeners.rbegin(), EventListeners.rend(), L); if (I != EventListeners.rend()) { std::swap(*I, EventListeners.back()); EventListeners.pop_back(); } } -void MCJIT::NotifyObjectEmitted(const ObjectImage& Obj) { + +void MCJIT::NotifyObjectEmitted(const object::ObjectFile& Obj, + const RuntimeDyld::LoadedObjectInfo &L) { MutexGuard locked(lock); - MemMgr.notifyObjectLoaded(this, &Obj); + MemMgr.notifyObjectLoaded(this, Obj); for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyObjectEmitted(Obj); + EventListeners[I]->NotifyObjectEmitted(Obj, L); } } -void MCJIT::NotifyFreeingObject(const ObjectImage& Obj) { + +void MCJIT::NotifyFreeingObject(const object::ObjectFile& Obj) { MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFreeingObject(Obj); - } + for (JITEventListener *L : EventListeners) + L->NotifyFreeingObject(Obj); } uint64_t LinkingMemoryManager::getSymbolAddress(const std::string &Name) { @@ -588,5 +588,7 @@ uint64_t LinkingMemoryManager::getSymbolAddress(const std::string &Name) { Result = ParentEngine->getSymbolAddress(Name.substr(1), false); if (Result) return Result; + if (ParentEngine->isSymbolSearchingDisabled()) + return 0; return ClientMM->getSymbolAddress(Name); } diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h index 100e9a23fcd3..f55dd60ecba9 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.h +++ b/lib/ExecutionEngine/MCJIT/MCJIT.h @@ -7,15 +7,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_EXECUTIONENGINE_MCJIT_H -#define LLVM_LIB_EXECUTIONENGINE_MCJIT_H +#ifndef LLVM_LIB_EXECUTIONENGINE_MCJIT_MCJIT_H +#define LLVM_LIB_EXECUTIONENGINE_MCJIT_MCJIT_H +#include "ObjectBuffer.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/ExecutionEngine/ObjectCache.h" -#include "llvm/ExecutionEngine/ObjectImage.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Module.h" @@ -28,8 +28,9 @@ class MCJIT; // to that object. class LinkingMemoryManager : public RTDyldMemoryManager { public: - LinkingMemoryManager(MCJIT *Parent, RTDyldMemoryManager *MM) - : ParentEngine(Parent), ClientMM(MM) {} + LinkingMemoryManager(MCJIT *Parent, + std::unique_ptr<RTDyldMemoryManager> MM) + : ParentEngine(Parent), ClientMM(std::move(MM)) {} uint64_t getSymbolAddress(const std::string &Name) override; @@ -57,7 +58,7 @@ public: } void notifyObjectLoaded(ExecutionEngine *EE, - const ObjectImage *Obj) override { + const object::ObjectFile &Obj) override { ClientMM->notifyObjectLoaded(EE, Obj); } @@ -101,8 +102,8 @@ private: // called. class MCJIT : public ExecutionEngine { - MCJIT(Module *M, TargetMachine *tm, RTDyldMemoryManager *MemMgr, - bool AllocateGVsWithCode); + MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm, + std::unique_ptr<RTDyldMemoryManager> MemMgr); typedef llvm::SmallPtrSet<Module *, 4> ModulePtrSet; @@ -118,6 +119,9 @@ class MCJIT : public ExecutionEngine { ModulePtrSet::iterator begin_added() { return AddedModules.begin(); } ModulePtrSet::iterator end_added() { return AddedModules.end(); } + iterator_range<ModulePtrSet::iterator> added() { + return iterator_range<ModulePtrSet::iterator>(begin_added(), end_added()); + } ModulePtrSet::iterator begin_loaded() { return LoadedModules.begin(); } ModulePtrSet::iterator end_loaded() { return LoadedModules.end(); } @@ -125,8 +129,8 @@ class MCJIT : public ExecutionEngine { ModulePtrSet::iterator begin_finalized() { return FinalizedModules.begin(); } ModulePtrSet::iterator end_finalized() { return FinalizedModules.end(); } - void addModule(Module *M) { - AddedModules.insert(M); + void addModule(std::unique_ptr<Module> M) { + AddedModules.insert(M.release()); } bool removeModule(Module *M) { @@ -208,18 +212,18 @@ class MCJIT : public ExecutionEngine { } }; - TargetMachine *TM; + std::unique_ptr<TargetMachine> TM; MCContext *Ctx; LinkingMemoryManager MemMgr; RuntimeDyld Dyld; - SmallVector<JITEventListener*, 2> EventListeners; + std::vector<JITEventListener*> EventListeners; OwningModuleContainer OwnedModules; - SmallVector<object::Archive*, 2> Archives; + SmallVector<object::OwningBinary<object::Archive>, 2> Archives; + SmallVector<std::unique_ptr<MemoryBuffer>, 2> Buffers; - typedef SmallVector<ObjectImage *, 2> LoadedObjectList; - LoadedObjectList LoadedObjects; + SmallVector<std::unique_ptr<object::ObjectFile>, 2> LoadedObjects; // An optional ObjectCache to be notified of compiled objects and used to // perform lookup of pre-compiled code to avoid re-compilation. @@ -238,9 +242,10 @@ public: /// @name ExecutionEngine interface implementation /// @{ - void addModule(Module *M) override; + void addModule(std::unique_ptr<Module> M) override; void addObjectFile(std::unique_ptr<object::ObjectFile> O) override; - void addArchive(object::Archive *O) override; + void addObjectFile(object::OwningBinary<object::ObjectFile> O) override; + void addArchive(object::OwningBinary<object::Archive> O) override; bool removeModule(Module *M) override; /// FindFunctionNamed - Search all of the active modules to find the one that @@ -276,14 +281,8 @@ public: /// \param isDtors - Run the destructors instead of constructors. void runStaticConstructorsDestructors(bool isDtors) override; - void *getPointerToBasicBlock(BasicBlock *BB) override; - void *getPointerToFunction(Function *F) override; - void *recompileAndRelinkFunction(Function *F) override; - - void freeMachineCodeForFunction(Function *F) override; - GenericValue runFunction(Function *F, const std::vector<GenericValue> &ArgValues) override; @@ -295,7 +294,7 @@ public: /// found, this function silently returns a null pointer. Otherwise, /// it prints a message to stderr and aborts. /// - void *getPointerToNamedFunction(const std::string &Name, + void *getPointerToNamedFunction(StringRef Name, bool AbortOnFailure = true) override; /// mapSectionAddress - map a section to its target address space value. @@ -315,7 +314,7 @@ public: uint64_t getGlobalValueAddress(const std::string &Name) override; uint64_t getFunctionAddress(const std::string &Name) override; - TargetMachine *getTargetMachine() override { return TM; } + TargetMachine *getTargetMachine() override { return TM.get(); } /// @} /// @name (Private) Registration Interfaces @@ -325,11 +324,10 @@ public: MCJITCtor = createJIT; } - static ExecutionEngine *createJIT(Module *M, + static ExecutionEngine *createJIT(std::unique_ptr<Module> M, std::string *ErrorStr, - RTDyldMemoryManager *MemMgr, - bool GVsWithCode, - TargetMachine *TM); + std::unique_ptr<RTDyldMemoryManager> MemMgr, + std::unique_ptr<TargetMachine> TM); // @} @@ -344,10 +342,11 @@ protected: /// this function call is expected to be the contained module. The module /// is passed as a parameter here to prepare for multiple module support in /// the future. - ObjectBufferStream* emitObject(Module *M); + std::unique_ptr<MemoryBuffer> emitObject(Module *M); - void NotifyObjectEmitted(const ObjectImage& Obj); - void NotifyFreeingObject(const ObjectImage& Obj); + void NotifyObjectEmitted(const object::ObjectFile& Obj, + const RuntimeDyld::LoadedObjectInfo &L); + void NotifyFreeingObject(const object::ObjectFile& Obj); uint64_t getExistingSymbolAddress(const std::string &Name); Module *findModuleForSymbol(const std::string &Name, diff --git a/lib/ExecutionEngine/MCJIT/ObjectBuffer.h b/lib/ExecutionEngine/MCJIT/ObjectBuffer.h new file mode 100644 index 000000000000..92310f3eb54a --- /dev/null +++ b/lib/ExecutionEngine/MCJIT/ObjectBuffer.h @@ -0,0 +1,48 @@ +//===--- ObjectBuffer.h - Utility class to wrap object memory ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares a wrapper class to hold the memory into which an +// object will be generated. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_OBJECTBUFFER_H +#define LLVM_EXECUTIONENGINE_OBJECTBUFFER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +class ObjectMemoryBuffer : public MemoryBuffer { +public: + template <unsigned N> + ObjectMemoryBuffer(SmallVector<char, N> SV) + : SV(SV), BufferName("<in-memory object>") { + init(this->SV.begin(), this->SV.end(), false); + } + + template <unsigned N> + ObjectMemoryBuffer(SmallVector<char, N> SV, StringRef Name) + : SV(SV), BufferName(Name) { + init(this->SV.begin(), this->SV.end(), false); + } + const char* getBufferIdentifier() const override { return BufferName.c_str(); } + + BufferKind getBufferKind() const override { return MemoryBuffer_Malloc; } + +private: + SmallVector<char, 4096> SV; + std::string BufferName; +}; + +} // namespace llvm + +#endif diff --git a/lib/ExecutionEngine/Makefile b/lib/ExecutionEngine/Makefile index c26e0ada5bc1..cf714324e3b6 100644 --- a/lib/ExecutionEngine/Makefile +++ b/lib/ExecutionEngine/Makefile @@ -11,7 +11,7 @@ LIBRARYNAME = LLVMExecutionEngine include $(LEVEL)/Makefile.config -PARALLEL_DIRS = Interpreter JIT MCJIT RuntimeDyld +PARALLEL_DIRS = Interpreter MCJIT RuntimeDyld ifeq ($(USE_INTEL_JITEVENTS), 1) PARALLEL_DIRS += IntelJITEvents diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp index fd37a13b112f..9ab40036bdf8 100644 --- a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp +++ b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp @@ -13,55 +13,50 @@ //===----------------------------------------------------------------------===// #include "llvm/Config/config.h" +#include "EventListenerCommon.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/ExecutionEngine/JITEventListener.h" - +#include "llvm/ExecutionEngine/OProfileWrapper.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Function.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/ExecutionEngine/ObjectImage.h" -#include "llvm/ExecutionEngine/OProfileWrapper.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/Errno.h" -#include "EventListenerCommon.h" - +#include "llvm/Support/raw_ostream.h" #include <dirent.h> #include <fcntl.h> using namespace llvm; using namespace llvm::jitprofiling; +using namespace llvm::object; #define DEBUG_TYPE "oprofile-jit-event-listener" namespace { class OProfileJITEventListener : public JITEventListener { - OProfileWrapper& Wrapper; + std::unique_ptr<OProfileWrapper> Wrapper; void initialize(); + std::map<const char*, OwningBinary<ObjectFile>> DebugObjects; public: - OProfileJITEventListener(OProfileWrapper& LibraryWrapper) - : Wrapper(LibraryWrapper) { + OProfileJITEventListener(std::unique_ptr<OProfileWrapper> LibraryWrapper) + : Wrapper(std::move(LibraryWrapper)) { initialize(); } ~OProfileJITEventListener(); - virtual void NotifyFunctionEmitted(const Function &F, - void *FnStart, size_t FnSize, - const JITEvent_EmittedFunctionDetails &Details); - - virtual void NotifyFreeingMachineCode(void *OldPtr); + void NotifyObjectEmitted(const ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &L) override; - virtual void NotifyObjectEmitted(const ObjectImage &Obj); - - virtual void NotifyFreeingObject(const ObjectImage &Obj); + void NotifyFreeingObject(const ObjectFile &Obj) override; }; void OProfileJITEventListener::initialize() { - if (!Wrapper.op_open_agent()) { + if (!Wrapper->op_open_agent()) { const std::string err_str = sys::StrError(); DEBUG(dbgs() << "Failed to connect to OProfile agent: " << err_str << "\n"); } else { @@ -70,8 +65,8 @@ void OProfileJITEventListener::initialize() { } OProfileJITEventListener::~OProfileJITEventListener() { - if (Wrapper.isAgentAvailable()) { - if (Wrapper.op_close_agent() == -1) { + if (Wrapper->isAgentAvailable()) { + if (Wrapper->op_close_agent() == -1) { const std::string err_str = sys::StrError(); DEBUG(dbgs() << "Failed to disconnect from OProfile agent: " << err_str << "\n"); @@ -81,101 +76,22 @@ OProfileJITEventListener::~OProfileJITEventListener() { } } -static debug_line_info LineStartToOProfileFormat( - const MachineFunction &MF, FilenameCache &Filenames, - uintptr_t Address, DebugLoc Loc) { - debug_line_info Result; - Result.vma = Address; - Result.lineno = Loc.getLine(); - Result.filename = Filenames.getFilename( - Loc.getScope(MF.getFunction()->getContext())); - DEBUG(dbgs() << "Mapping " << reinterpret_cast<void*>(Result.vma) << " to " - << Result.filename << ":" << Result.lineno << "\n"); - return Result; -} - -// Adds the just-emitted function to the symbol table. -void OProfileJITEventListener::NotifyFunctionEmitted( - const Function &F, void *FnStart, size_t FnSize, - const JITEvent_EmittedFunctionDetails &Details) { - assert(F.hasName() && FnStart != 0 && "Bad symbol to add"); - if (Wrapper.op_write_native_code(F.getName().data(), - reinterpret_cast<uint64_t>(FnStart), - FnStart, FnSize) == -1) { - DEBUG(dbgs() << "Failed to tell OProfile about native function " - << F.getName() << " at [" - << FnStart << "-" << ((char*)FnStart + FnSize) << "]\n"); +void OProfileJITEventListener::NotifyObjectEmitted( + const ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &L) { + if (!Wrapper->isAgentAvailable()) { return; } - if (!Details.LineStarts.empty()) { - // Now we convert the line number information from the address/DebugLoc - // format in Details to the address/filename/lineno format that OProfile - // expects. Note that OProfile 0.9.4 has a bug that causes it to ignore - // line numbers for addresses above 4G. - FilenameCache Filenames; - std::vector<debug_line_info> LineInfo; - LineInfo.reserve(1 + Details.LineStarts.size()); - - DebugLoc FirstLoc = Details.LineStarts[0].Loc; - assert(!FirstLoc.isUnknown() - && "LineStarts should not contain unknown DebugLocs"); - MDNode *FirstLocScope = FirstLoc.getScope(F.getContext()); - DISubprogram FunctionDI = getDISubprogram(FirstLocScope); - if (FunctionDI.Verify()) { - // If we have debug info for the function itself, use that as the line - // number of the first several instructions. Otherwise, after filling - // LineInfo, we'll adjust the address of the first line number to point at - // the start of the function. - debug_line_info line_info; - line_info.vma = reinterpret_cast<uintptr_t>(FnStart); - line_info.lineno = FunctionDI.getLineNumber(); - line_info.filename = Filenames.getFilename(FirstLocScope); - LineInfo.push_back(line_info); - } - - for (std::vector<EmittedFunctionDetails::LineStart>::const_iterator - I = Details.LineStarts.begin(), E = Details.LineStarts.end(); - I != E; ++I) { - LineInfo.push_back(LineStartToOProfileFormat( - *Details.MF, Filenames, I->Address, I->Loc)); - } - - // In case the function didn't have line info of its own, adjust the first - // line info's address to include the start of the function. - LineInfo[0].vma = reinterpret_cast<uintptr_t>(FnStart); - - if (Wrapper.op_write_debug_line_info(FnStart, LineInfo.size(), - &*LineInfo.begin()) == -1) { - DEBUG(dbgs() - << "Failed to tell OProfile about line numbers for native function " - << F.getName() << " at [" - << FnStart << "-" << ((char*)FnStart + FnSize) << "]\n"); - } - } -} - -// Removes the being-deleted function from the symbol table. -void OProfileJITEventListener::NotifyFreeingMachineCode(void *FnStart) { - assert(FnStart && "Invalid function pointer"); - if (Wrapper.op_unload_native_code(reinterpret_cast<uint64_t>(FnStart)) == -1) { - DEBUG(dbgs() - << "Failed to tell OProfile about unload of native function at " - << FnStart << "\n"); - } -} - -void OProfileJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { - if (!Wrapper.isAgentAvailable()) { - return; - } + OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj); + const ObjectFile &DebugObj = *DebugObjOwner.getBinary(); // Use symbol info to iterate functions in the object. - for (object::symbol_iterator I = Obj.begin_symbols(), E = Obj.end_symbols(); + for (symbol_iterator I = DebugObj.symbol_begin(), E = DebugObj.symbol_end(); I != E; ++I) { - object::SymbolRef::Type SymType; + SymbolRef::Type SymType; if (I->getType(SymType)) continue; - if (SymType == object::SymbolRef::ST_Function) { + if (SymType == SymbolRef::ST_Function) { StringRef Name; uint64_t Addr; uint64_t Size; @@ -183,7 +99,7 @@ void OProfileJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { if (I->getAddress(Addr)) continue; if (I->getSize(Size)) continue; - if (Wrapper.op_write_native_code(Name.data(), Addr, (void*)Addr, Size) + if (Wrapper->op_write_native_code(Name.data(), Addr, (void*)Addr, Size) == -1) { DEBUG(dbgs() << "Failed to tell OProfile about native function " << Name << " at [" @@ -193,45 +109,48 @@ void OProfileJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { // TODO: support line number info (similar to IntelJITEventListener.cpp) } } + + DebugObjects[Obj.getData().data()] = std::move(DebugObjOwner); } -void OProfileJITEventListener::NotifyFreeingObject(const ObjectImage &Obj) { - if (!Wrapper.isAgentAvailable()) { - return; - } +void OProfileJITEventListener::NotifyFreeingObject(const ObjectFile &Obj) { + if (Wrapper->isAgentAvailable()) { - // Use symbol info to iterate functions in the object. - for (object::symbol_iterator I = Obj.begin_symbols(), E = Obj.end_symbols(); - I != E; ++I) { - object::SymbolRef::Type SymType; - if (I->getType(SymType)) continue; - if (SymType == object::SymbolRef::ST_Function) { - uint64_t Addr; - if (I->getAddress(Addr)) continue; + // If there was no agent registered when the original object was loaded then + // we won't have created a debug object for it, so bail out. + if (DebugObjects.find(Obj.getData().data()) == DebugObjects.end()) + return; - if (Wrapper.op_unload_native_code(Addr) == -1) { - DEBUG(dbgs() - << "Failed to tell OProfile about unload of native function at " - << (void*)Addr << "\n"); - continue; + const ObjectFile &DebugObj = *DebugObjects[Obj.getData().data()].getBinary(); + + // Use symbol info to iterate functions in the object. + for (symbol_iterator I = DebugObj.symbol_begin(), + E = DebugObj.symbol_end(); + I != E; ++I) { + SymbolRef::Type SymType; + if (I->getType(SymType)) continue; + if (SymType == SymbolRef::ST_Function) { + uint64_t Addr; + if (I->getAddress(Addr)) continue; + + if (Wrapper->op_unload_native_code(Addr) == -1) { + DEBUG(dbgs() + << "Failed to tell OProfile about unload of native function at " + << (void*)Addr << "\n"); + continue; + } } } } + + DebugObjects.erase(Obj.getData().data()); } } // anonymous namespace. namespace llvm { JITEventListener *JITEventListener::createOProfileJITEventListener() { - static std::unique_ptr<OProfileWrapper> JITProfilingWrapper( - new OProfileWrapper); - return new OProfileJITEventListener(*JITProfilingWrapper); -} - -// for testing -JITEventListener *JITEventListener::createOProfileJITEventListener( - OProfileWrapper* TestImpl) { - return new OProfileJITEventListener(*TestImpl); + return new OProfileJITEventListener(llvm::make_unique<OProfileWrapper>()); } } // namespace llvm diff --git a/lib/ExecutionEngine/RTDyldMemoryManager.cpp b/lib/ExecutionEngine/RTDyldMemoryManager.cpp index 1646937e816c..2a5e4f83228b 100644 --- a/lib/ExecutionEngine/RTDyldMemoryManager.cpp +++ b/lib/ExecutionEngine/RTDyldMemoryManager.cpp @@ -13,6 +13,7 @@ #include "llvm/Config/config.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/ErrorHandling.h" #include <cstdlib> @@ -210,7 +211,13 @@ ARM_MATH_IMPORTS(ARM_MATH_DECL) #undef ARM_MATH_DECL #endif -uint64_t RTDyldMemoryManager::getSymbolAddress(const std::string &Name) { +#if defined(__linux__) && defined(__GLIBC__) && \ + (defined(__i386__) || defined(__x86_64__)) +extern "C" LLVM_ATTRIBUTE_WEAK void __morestack(); +#endif + +uint64_t +RTDyldMemoryManager::getSymbolAddressInProcess(const std::string &Name) { // This implementation assumes that the host program is the target. // Clients generating code for a remote target should implement their own // memory manager. @@ -232,6 +239,12 @@ uint64_t RTDyldMemoryManager::getSymbolAddress(const std::string &Name) { if (Name == "lstat64") return (uint64_t)&lstat64; if (Name == "atexit") return (uint64_t)&atexit; if (Name == "mknod") return (uint64_t)&mknod; + +#if defined(__i386__) || defined(__x86_64__) + // __morestack lives in libgcc, a static library. + if (&__morestack && Name == "__morestack") + return (uint64_t)&__morestack; +#endif #endif // __linux__ && __GLIBC__ // See ARM_MATH_IMPORTS definition for explanation @@ -253,19 +266,19 @@ uint64_t RTDyldMemoryManager::getSymbolAddress(const std::string &Name) { // is called before ExecutionEngine::runFunctionAsMain() is called. if (Name == "__main") return (uint64_t)&jit_noop; + // Try to demangle Name before looking it up in the process, otherwise symbol + // '_<Name>' (if present) will shadow '<Name>', and there will be no way to + // refer to the latter. + const char *NameStr = Name.c_str(); - void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); - if (Ptr) - return (uint64_t)Ptr; - - // If it wasn't found and if it starts with an underscore ('_') character, - // try again without the underscore. - if (NameStr[0] == '_') { - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); - if (Ptr) + + if (NameStr[0] == '_') + if (void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr + 1)) return (uint64_t)Ptr; - } - return 0; + + // If we Name did not require demangling, or we failed to find the demangled + // name, try again without demangling. + return (uint64_t)sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); } void *RTDyldMemoryManager::getPointerToNamedFunction(const std::string &Name, diff --git a/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt b/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt index eb1a60b60d08..5790eee9c822 100644 --- a/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt +++ b/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt @@ -1,5 +1,4 @@ add_llvm_library(LLVMRuntimeDyld - GDBRegistrar.cpp RuntimeDyld.cpp RuntimeDyldChecker.cpp RuntimeDyldELF.cpp diff --git a/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h b/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h deleted file mode 100644 index 6a514ea3ec3b..000000000000 --- a/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h +++ /dev/null @@ -1,44 +0,0 @@ -//===-- JITRegistrar.h - Registers objects with a debugger ----------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H -#define LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H - -#include "llvm/ExecutionEngine/ObjectBuffer.h" - -namespace llvm { - -/// Global access point for the JIT debugging interface. -class JITRegistrar { - virtual void anchor(); -public: - /// Instantiates the JIT service. - JITRegistrar() {} - - /// Unregisters each object that was previously registered and releases all - /// internal resources. - virtual ~JITRegistrar() {} - - /// Creates an entry in the JIT registry for the buffer @p Object, - /// which must contain an object file in executable memory with any - /// debug information for the debugger. - virtual void registerObject(const ObjectBuffer &Object) = 0; - - /// Removes the internal registration of @p Object, and - /// frees associated resources. - /// Returns true if @p Object was previously registered. - virtual bool deregisterObject(const ObjectBuffer &Object) = 0; - - /// Returns a reference to a GDB JIT registrar singleton - static JITRegistrar& getGDBRegistrar(); -}; - -} // end namespace llvm - -#endif // LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H diff --git a/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h b/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h deleted file mode 100644 index c3a21823bbc8..000000000000 --- a/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h +++ /dev/null @@ -1,89 +0,0 @@ -//===-- ObjectImageCommon.h - Format independent executuable object image -===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file declares a file format independent ObjectImage class. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_RUNTIMEDYLD_OBJECTIMAGECOMMON_H -#define LLVM_RUNTIMEDYLD_OBJECTIMAGECOMMON_H - -#include "llvm/ExecutionEngine/ObjectBuffer.h" -#include "llvm/ExecutionEngine/ObjectImage.h" -#include "llvm/Object/ObjectFile.h" - -#include <memory> - -namespace llvm { - -namespace object { - class ObjectFile; -} - -class ObjectImageCommon : public ObjectImage { - ObjectImageCommon(); // = delete - ObjectImageCommon(const ObjectImageCommon &other); // = delete - void anchor() override; - -protected: - std::unique_ptr<object::ObjectFile> ObjFile; - - // This form of the constructor allows subclasses to use - // format-specific subclasses of ObjectFile directly - ObjectImageCommon(ObjectBuffer *Input, std::unique_ptr<object::ObjectFile> Obj) - : ObjectImage(Input), // saves Input as Buffer and takes ownership - ObjFile(std::move(Obj)) - { - } - -public: - ObjectImageCommon(ObjectBuffer* Input) - : ObjectImage(Input) // saves Input as Buffer and takes ownership - { - // FIXME: error checking? createObjectFile returns an ErrorOr<ObjectFile*> - // and should probably be checked for failure. - std::unique_ptr<MemoryBuffer> Buf(Buffer->getMemBuffer()); - ObjFile.reset(object::ObjectFile::createObjectFile(Buf).get()); - } - ObjectImageCommon(std::unique_ptr<object::ObjectFile> Input) - : ObjectImage(nullptr), ObjFile(std::move(Input)) {} - virtual ~ObjectImageCommon() { } - - object::symbol_iterator begin_symbols() const override - { return ObjFile->symbol_begin(); } - object::symbol_iterator end_symbols() const override - { return ObjFile->symbol_end(); } - - object::section_iterator begin_sections() const override - { return ObjFile->section_begin(); } - object::section_iterator end_sections() const override - { return ObjFile->section_end(); } - - /* Triple::ArchType */ unsigned getArch() const override - { return ObjFile->getArch(); } - - StringRef getData() const override { return ObjFile->getData(); } - - object::ObjectFile* getObjectFile() const override { return ObjFile.get(); } - - // Subclasses can override these methods to update the image with loaded - // addresses for sections and common symbols - void updateSectionAddress(const object::SectionRef &Sec, - uint64_t Addr) override {} - void updateSymbolAddress(const object::SymbolRef &Sym, - uint64_t Addr) override {} - - // Subclasses can override these methods to provide JIT debugging support - void registerWithDebugger() override {} - void deregisterWithDebugger() override {} -}; - -} // end namespace llvm - -#endif // LLVM_RUNTIMEDYLD_OBJECT_IMAGE_H diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index d86a75130685..304014edb247 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -12,12 +12,11 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/RuntimeDyld.h" -#include "JITRegistrar.h" -#include "ObjectImageCommon.h" +#include "RuntimeDyldCheckerImpl.h" #include "RuntimeDyldELF.h" #include "RuntimeDyldImpl.h" #include "RuntimeDyldMachO.h" -#include "llvm/Object/ELF.h" +#include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/MutexGuard.h" @@ -29,10 +28,8 @@ using namespace llvm::object; // Empty out-of-line virtual destructor as the key function. RuntimeDyldImpl::~RuntimeDyldImpl() {} -// Pin the JITRegistrar's and ObjectImage*'s vtables to this file. -void JITRegistrar::anchor() {} -void ObjectImage::anchor() {} -void ObjectImageCommon::anchor() {} +// Pin LoadedObjectInfo's vtables to this file. +void RuntimeDyld::LoadedObjectInfo::anchor() {} namespace llvm { @@ -40,6 +37,44 @@ void RuntimeDyldImpl::registerEHFrames() {} void RuntimeDyldImpl::deregisterEHFrames() {} +#ifndef NDEBUG +static void dumpSectionMemory(const SectionEntry &S, StringRef State) { + dbgs() << "----- Contents of section " << S.Name << " " << State << " -----"; + + if (S.Address == nullptr) { + dbgs() << "\n <section not emitted>\n"; + return; + } + + const unsigned ColsPerRow = 16; + + uint8_t *DataAddr = S.Address; + uint64_t LoadAddr = S.LoadAddress; + + unsigned StartPadding = LoadAddr & (ColsPerRow - 1); + unsigned BytesRemaining = S.Size; + + if (StartPadding) { + dbgs() << "\n" << format("0x%016" PRIx64, LoadAddr & ~(ColsPerRow - 1)) << ":"; + while (StartPadding--) + dbgs() << " "; + } + + while (BytesRemaining > 0) { + if ((LoadAddr & (ColsPerRow - 1)) == 0) + dbgs() << "\n" << format("0x%016" PRIx64, LoadAddr) << ":"; + + dbgs() << " " << format("%02x", *DataAddr); + + ++DataAddr; + ++LoadAddr; + --BytesRemaining; + } + + dbgs() << "\n"; +} +#endif + // Resolve the relocations for all symbols we currently know about. void RuntimeDyldImpl::resolveRelocations() { MutexGuard locked(lock); @@ -55,8 +90,10 @@ void RuntimeDyldImpl::resolveRelocations() { // entry provides the section to which the relocation will be applied. uint64_t Addr = Sections[i].LoadAddress; DEBUG(dbgs() << "Resolving relocations Section #" << i << "\t" - << format("%p", (uint8_t *)Addr) << "\n"); + << format("0x%x", Addr) << "\n"); + DEBUG(dumpSectionMemory(Sections[i], "before relocations")); resolveRelocationList(Relocations[i], Addr); + DEBUG(dumpSectionMemory(Sections[i], "after relocations")); Relocations.erase(i); } } @@ -88,40 +125,36 @@ static std::error_code getOffset(const SymbolRef &Sym, uint64_t &Result) { if (std::error_code EC = Sym.getSection(SecI)) return EC; - if (SecI == Obj->section_end()) { - Result = UnknownAddressOrSize; - return object_error::success; - } - - uint64_t SectionAddress; - if (std::error_code EC = SecI->getAddress(SectionAddress)) - return EC; + if (SecI == Obj->section_end()) { + Result = UnknownAddressOrSize; + return object_error::success; + } + uint64_t SectionAddress = SecI->getAddress(); Result = Address - SectionAddress; return object_error::success; } -ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { +std::pair<unsigned, unsigned> +RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { MutexGuard locked(lock); - std::unique_ptr<ObjectImage> Obj(InputObject); - if (!Obj) - return nullptr; + // Grab the first Section ID. We'll use this later to construct the underlying + // range for the returned LoadedObjectInfo. + unsigned SectionsAddedBeginIdx = Sections.size(); // Save information about our target - Arch = (Triple::ArchType)Obj->getArch(); - IsTargetLittleEndian = Obj->getObjectFile()->isLittleEndian(); + Arch = (Triple::ArchType)Obj.getArch(); + IsTargetLittleEndian = Obj.isLittleEndian(); // Compute the memory size required to load all sections to be loaded // and pass this information to the memory manager if (MemMgr->needsToReserveAllocationSpace()) { uint64_t CodeSize = 0, DataSizeRO = 0, DataSizeRW = 0; - computeTotalAllocSize(*Obj, CodeSize, DataSizeRO, DataSizeRW); + computeTotalAllocSize(Obj, CodeSize, DataSizeRO, DataSizeRW); MemMgr->reserveAllocationSpace(CodeSize, DataSizeRO, DataSizeRW); } - // Symbols found in this object - StringMap<SymbolLoc> LocalSymbols; // Used sections from the object file ObjSectionToIDMap LocalSections; @@ -132,7 +165,7 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { // Parse symbols DEBUG(dbgs() << "Parse symbols:\n"); - for (symbol_iterator I = Obj->begin_symbols(), E = Obj->end_symbols(); I != E; + for (symbol_iterator I = Obj.symbol_begin(), E = Obj.symbol_end(); I != E; ++I) { object::SymbolRef::Type SymType; StringRef Name; @@ -158,17 +191,15 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { SymType == object::SymbolRef::ST_Unknown) { uint64_t SectOffset; StringRef SectionData; - bool IsCode; - section_iterator SI = Obj->end_sections(); + section_iterator SI = Obj.section_end(); Check(getOffset(*I, SectOffset)); Check(I->getSection(SI)); - if (SI == Obj->end_sections()) + if (SI == Obj.section_end()) continue; Check(SI->getContents(SectionData)); - Check(SI->isText(IsCode)); + bool IsCode = SI->isText(); unsigned SectionID = - findOrEmitSection(*Obj, *SI, IsCode, LocalSections); - LocalSymbols[Name.data()] = SymbolLoc(SectionID, SectOffset); + findOrEmitSection(Obj, *SI, IsCode, LocalSections); DEBUG(dbgs() << "\tOffset: " << format("%p", (uintptr_t)SectOffset) << " flags: " << Flags << " SID: " << SectionID); GlobalSymbolTable[Name] = SymbolLoc(SectionID, SectOffset); @@ -179,11 +210,11 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { // Allocate common symbols if (CommonSize != 0) - emitCommonSymbols(*Obj, CommonSymbols, CommonSize, GlobalSymbolTable); + emitCommonSymbols(Obj, CommonSymbols, CommonSize, GlobalSymbolTable); // Parse and process relocations DEBUG(dbgs() << "Parse relocations:\n"); - for (section_iterator SI = Obj->begin_sections(), SE = Obj->end_sections(); + for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); SI != SE; ++SI) { unsigned SectionID = 0; StubMap Stubs; @@ -195,21 +226,26 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { if (I == E && !ProcessAllSections) continue; - bool IsCode = false; - Check(RelocatedSection->isText(IsCode)); + bool IsCode = RelocatedSection->isText(); SectionID = - findOrEmitSection(*Obj, *RelocatedSection, IsCode, LocalSections); + findOrEmitSection(Obj, *RelocatedSection, IsCode, LocalSections); DEBUG(dbgs() << "\tSectionID: " << SectionID << "\n"); for (; I != E;) - I = processRelocationRef(SectionID, I, *Obj, LocalSections, LocalSymbols, - Stubs); + I = processRelocationRef(SectionID, I, Obj, LocalSections, Stubs); + + // If there is an attached checker, notify it about the stubs for this + // section so that they can be verified. + if (Checker) + Checker->registerStubMap(Obj.getFileName(), SectionID, Stubs); } // Give the subclasses a chance to tie-up any loose ends. - finalizeLoad(*Obj, LocalSections); + finalizeLoad(Obj, LocalSections); - return Obj.release(); + unsigned SectionsAddedEndIdx = Sections.size(); + + return std::make_pair(SectionsAddedBeginIdx, SectionsAddedEndIdx); } // A helper method for computeTotalAllocSize. @@ -227,9 +263,37 @@ computeAllocationSizeForSections(std::vector<uint64_t> &SectionSizes, return TotalSize; } +static bool isRequiredForExecution(const SectionRef &Section) { + const ObjectFile *Obj = Section.getObject(); + if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj)) + return ELFObj->getSectionFlags(Section) & ELF::SHF_ALLOC; + assert(isa<MachOObjectFile>(Obj)); + return true; + } + +static bool isReadOnlyData(const SectionRef &Section) { + const ObjectFile *Obj = Section.getObject(); + if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj)) + return !(ELFObj->getSectionFlags(Section) & + (ELF::SHF_WRITE | ELF::SHF_EXECINSTR)); + assert(isa<MachOObjectFile>(Obj)); + return false; +} + +static bool isZeroInit(const SectionRef &Section) { + const ObjectFile *Obj = Section.getObject(); + if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj)) + return ELFObj->getSectionType(Section) == ELF::SHT_NOBITS; + + auto *MachO = cast<MachOObjectFile>(Obj); + unsigned SectionType = MachO->getSectionType(Section); + return SectionType == MachO::S_ZEROFILL || + SectionType == MachO::S_GB_ZEROFILL; +} + // Compute an upper bound of the memory size that is required to load all // sections -void RuntimeDyldImpl::computeTotalAllocSize(ObjectImage &Obj, +void RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, uint64_t &CodeSize, uint64_t &DataSizeRO, uint64_t &DataSizeRW) { @@ -241,24 +305,19 @@ void RuntimeDyldImpl::computeTotalAllocSize(ObjectImage &Obj, // Collect sizes of all sections to be loaded; // also determine the max alignment of all sections - for (section_iterator SI = Obj.begin_sections(), SE = Obj.end_sections(); + for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); SI != SE; ++SI) { const SectionRef &Section = *SI; - bool IsRequired; - Check(Section.isRequiredForExecution(IsRequired)); + bool IsRequired = isRequiredForExecution(Section); // Consider only the sections that are required to be loaded for execution if (IsRequired) { - uint64_t DataSize = 0; - uint64_t Alignment64 = 0; - bool IsCode = false; - bool IsReadOnly = false; StringRef Name; - Check(Section.getSize(DataSize)); - Check(Section.getAlignment(Alignment64)); - Check(Section.isText(IsCode)); - Check(Section.isReadOnlyData(IsReadOnly)); + uint64_t DataSize = Section.getSize(); + uint64_t Alignment64 = Section.getAlignment(); + bool IsCode = Section.isText(); + bool IsReadOnly = isReadOnlyData(Section); Check(Section.getName(Name)); unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; @@ -292,7 +351,7 @@ void RuntimeDyldImpl::computeTotalAllocSize(ObjectImage &Obj, // Compute the size of all common symbols uint64_t CommonSize = 0; - for (symbol_iterator I = Obj.begin_symbols(), E = Obj.end_symbols(); I != E; + for (symbol_iterator I = Obj.symbol_begin(), E = Obj.symbol_end(); I != E; ++I) { uint32_t Flags = I->getFlags(); if (Flags & SymbolRef::SF_Common) { @@ -317,7 +376,7 @@ void RuntimeDyldImpl::computeTotalAllocSize(ObjectImage &Obj, } // compute stub buffer size for the given section -unsigned RuntimeDyldImpl::computeSectionStubBufSize(ObjectImage &Obj, +unsigned RuntimeDyldImpl::computeSectionStubBufSize(const ObjectFile &Obj, const SectionRef &Section) { unsigned StubSize = getMaxStubSize(); if (StubSize == 0) { @@ -327,7 +386,7 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(ObjectImage &Obj, // necessary section allocation size in loadObject by walking all the sections // once. unsigned StubBufSize = 0; - for (section_iterator SI = Obj.begin_sections(), SE = Obj.end_sections(); + for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); SI != SE; ++SI) { section_iterator RelSecI = SI->getRelocatedSection(); if (!(RelSecI == Section)) @@ -340,10 +399,8 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(ObjectImage &Obj, } // Get section data size and alignment - uint64_t Alignment64; - uint64_t DataSize; - Check(Section.getSize(DataSize)); - Check(Section.getAlignment(Alignment64)); + uint64_t DataSize = Section.getSize(); + uint64_t Alignment64 = Section.getAlignment(); // Add stubbuf size alignment unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; @@ -354,7 +411,37 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(ObjectImage &Obj, return StubBufSize; } -void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj, +uint64_t RuntimeDyldImpl::readBytesUnaligned(uint8_t *Src, + unsigned Size) const { + uint64_t Result = 0; + if (IsTargetLittleEndian) { + Src += Size - 1; + while (Size--) + Result = (Result << 8) | *Src--; + } else + while (Size--) + Result = (Result << 8) | *Src++; + + return Result; +} + +void RuntimeDyldImpl::writeBytesUnaligned(uint64_t Value, uint8_t *Dst, + unsigned Size) const { + if (IsTargetLittleEndian) { + while (Size--) { + *Dst++ = Value & 0xFF; + Value >>= 8; + } + } else { + Dst += Size - 1; + while (Size--) { + *Dst-- = Value & 0xFF; + Value >>= 8; + } + } +} + +void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, const CommonSymbolMap &CommonSymbols, uint64_t TotalSize, SymbolTableMap &SymbolTable) { @@ -365,7 +452,7 @@ void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj, if (!Addr) report_fatal_error("Unable to allocate memory for common symbols!"); uint64_t Offset = 0; - Sections.push_back(SectionEntry(StringRef(), Addr, TotalSize, 0)); + Sections.push_back(SectionEntry("<common symbols>", Addr, TotalSize, 0)); memset(Addr, 0, TotalSize); DEBUG(dbgs() << "emitCommonSection SectionID: " << SectionID << " new addr: " @@ -386,35 +473,28 @@ void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj, DEBUG(dbgs() << "Allocating common symbol " << Name << " address " << format("%p\n", Addr)); } - Obj.updateSymbolAddress(it->first, (uint64_t)Addr); SymbolTable[Name.data()] = SymbolLoc(SectionID, Offset); Offset += Size; Addr += Size; } } -unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj, +unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj, const SectionRef &Section, bool IsCode) { StringRef data; - uint64_t Alignment64; Check(Section.getContents(data)); - Check(Section.getAlignment(Alignment64)); + uint64_t Alignment64 = Section.getAlignment(); unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; - bool IsRequired; - bool IsVirtual; - bool IsZeroInit; - bool IsReadOnly; - uint64_t DataSize; unsigned PaddingSize = 0; unsigned StubBufSize = 0; StringRef Name; - Check(Section.isRequiredForExecution(IsRequired)); - Check(Section.isVirtual(IsVirtual)); - Check(Section.isZeroInit(IsZeroInit)); - Check(Section.isReadOnlyData(IsReadOnly)); - Check(Section.getSize(DataSize)); + bool IsRequired = isRequiredForExecution(Section); + bool IsVirtual = Section.isVirtual(); + bool IsZeroInit = isZeroInit(Section); + bool IsReadOnly = isReadOnlyData(Section); + uint64_t DataSize = Section.getSize(); Check(Section.getName(Name)); StubBufSize = computeSectionStubBufSize(Obj, Section); @@ -463,7 +543,6 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj, << " new addr: " << format("%p", Addr) << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"); - Obj.updateSectionAddress(Section, (uint64_t)Addr); } else { // Even if we didn't load the section, we need to record an entry for it // to handle later processing (and by 'handle' I mean don't do anything @@ -477,10 +556,14 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj, } Sections.push_back(SectionEntry(Name, Addr, DataSize, (uintptr_t)pData)); + + if (Checker) + Checker->registerSection(Obj.getFileName(), SectionID); + return SectionID; } -unsigned RuntimeDyldImpl::findOrEmitSection(ObjectImage &Obj, +unsigned RuntimeDyldImpl::findOrEmitSection(const ObjectFile &Obj, const SectionRef &Section, bool IsCode, ObjSectionToIDMap &LocalSections) { @@ -519,33 +602,24 @@ void RuntimeDyldImpl::addRelocationForSymbol(const RelocationEntry &RE, uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr, unsigned AbiVariant) { - if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be || - Arch == Triple::arm64 || Arch == Triple::arm64_be) { + if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) { // This stub has to be able to access the full address space, // since symbol lookup won't necessarily find a handy, in-range, // PLT stub for functions which could be anywhere. - uint32_t *StubAddr = (uint32_t *)Addr; - // Stub can use ip0 (== x16) to calculate address - *StubAddr = 0xd2e00010; // movz ip0, #:abs_g3:<addr> - StubAddr++; - *StubAddr = 0xf2c00010; // movk ip0, #:abs_g2_nc:<addr> - StubAddr++; - *StubAddr = 0xf2a00010; // movk ip0, #:abs_g1_nc:<addr> - StubAddr++; - *StubAddr = 0xf2800010; // movk ip0, #:abs_g0_nc:<addr> - StubAddr++; - *StubAddr = 0xd61f0200; // br ip0 + writeBytesUnaligned(0xd2e00010, Addr, 4); // movz ip0, #:abs_g3:<addr> + writeBytesUnaligned(0xf2c00010, Addr+4, 4); // movk ip0, #:abs_g2_nc:<addr> + writeBytesUnaligned(0xf2a00010, Addr+8, 4); // movk ip0, #:abs_g1_nc:<addr> + writeBytesUnaligned(0xf2800010, Addr+12, 4); // movk ip0, #:abs_g0_nc:<addr> + writeBytesUnaligned(0xd61f0200, Addr+16, 4); // br ip0 return Addr; } else if (Arch == Triple::arm || Arch == Triple::armeb) { // TODO: There is only ARM far stub now. We should add the Thumb stub, // and stubs for branches Thumb - ARM and ARM - Thumb. - uint32_t *StubAddr = (uint32_t *)Addr; - *StubAddr = 0xe51ff004; // ldr pc,<label> - return (uint8_t *)++StubAddr; + writeBytesUnaligned(0xe51ff004, Addr, 4); // ldr pc,<label> + return Addr + 4; } else if (Arch == Triple::mipsel || Arch == Triple::mips) { - uint32_t *StubAddr = (uint32_t *)Addr; // 0: 3c190000 lui t9,%hi(addr). // 4: 27390000 addiu t9,t9,%lo(addr). // 8: 03200008 jr t9. @@ -553,13 +627,10 @@ uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr, const unsigned LuiT9Instr = 0x3c190000, AdduiT9Instr = 0x27390000; const unsigned JrT9Instr = 0x03200008, NopInstr = 0x0; - *StubAddr = LuiT9Instr; - StubAddr++; - *StubAddr = AdduiT9Instr; - StubAddr++; - *StubAddr = JrT9Instr; - StubAddr++; - *StubAddr = NopInstr; + writeBytesUnaligned(LuiT9Instr, Addr, 4); + writeBytesUnaligned(AdduiT9Instr, Addr+4, 4); + writeBytesUnaligned(JrT9Instr, Addr+8, 4); + writeBytesUnaligned(NopInstr, Addr+12, 4); return Addr; } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) { // Depending on which version of the ELF ABI is in use, we need to @@ -619,6 +690,10 @@ void RuntimeDyldImpl::reassignSectionAddress(unsigned SectionID, // Addr is a uint64_t because we can't assume the pointer width // of the target is the same as that of the host. Just use a generic // "big enough" type. + DEBUG(dbgs() << "Reassigning address for section " + << SectionID << " (" << Sections[SectionID].Name << "): " + << format("0x%016" PRIx64, Sections[SectionID].LoadAddress) << " -> " + << format("0x%016" PRIx64, Addr) << "\n"); Sections[SectionID].LoadAddress = Addr; } @@ -685,6 +760,16 @@ void RuntimeDyldImpl::resolveExternalSymbols() { //===----------------------------------------------------------------------===// // RuntimeDyld class implementation + +uint64_t RuntimeDyld::LoadedObjectInfo::getSectionLoadAddress( + StringRef SectionName) const { + for (unsigned I = BeginIdx; I != EndIdx; ++I) + if (RTDyld.Sections[I].Name == SectionName) + return RTDyld.Sections[I].LoadAddress; + + return 0; +} + RuntimeDyld::RuntimeDyld(RTDyldMemoryManager *mm) { // FIXME: There's a potential issue lurking here if a single instance of // RuntimeDyld is used to load multiple objects. The current implementation @@ -695,104 +780,55 @@ RuntimeDyld::RuntimeDyld(RTDyldMemoryManager *mm) { Dyld = nullptr; MM = mm; ProcessAllSections = false; + Checker = nullptr; } -RuntimeDyld::~RuntimeDyld() { delete Dyld; } +RuntimeDyld::~RuntimeDyld() {} static std::unique_ptr<RuntimeDyldELF> -createRuntimeDyldELF(RTDyldMemoryManager *MM, bool ProcessAllSections) { +createRuntimeDyldELF(RTDyldMemoryManager *MM, bool ProcessAllSections, + RuntimeDyldCheckerImpl *Checker) { std::unique_ptr<RuntimeDyldELF> Dyld(new RuntimeDyldELF(MM)); Dyld->setProcessAllSections(ProcessAllSections); + Dyld->setRuntimeDyldChecker(Checker); return Dyld; } static std::unique_ptr<RuntimeDyldMachO> createRuntimeDyldMachO(Triple::ArchType Arch, RTDyldMemoryManager *MM, - bool ProcessAllSections) { + bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { std::unique_ptr<RuntimeDyldMachO> Dyld(RuntimeDyldMachO::create(Arch, MM)); Dyld->setProcessAllSections(ProcessAllSections); + Dyld->setRuntimeDyldChecker(Checker); return Dyld; } -ObjectImage *RuntimeDyld::loadObject(std::unique_ptr<ObjectFile> InputObject) { - std::unique_ptr<ObjectImage> InputImage; - - ObjectFile &Obj = *InputObject; - - if (InputObject->isELF()) { - InputImage.reset(RuntimeDyldELF::createObjectImageFromFile(std::move(InputObject))); - if (!Dyld) - Dyld = createRuntimeDyldELF(MM, ProcessAllSections).release(); - } else if (InputObject->isMachO()) { - InputImage.reset(RuntimeDyldMachO::createObjectImageFromFile(std::move(InputObject))); - if (!Dyld) - Dyld = createRuntimeDyldMachO( - static_cast<Triple::ArchType>(InputImage->getArch()), - MM, ProcessAllSections).release(); - } else - report_fatal_error("Incompatible object format!"); - - if (!Dyld->isCompatibleFile(&Obj)) - report_fatal_error("Incompatible object format!"); - - Dyld->loadObject(InputImage.get()); - return InputImage.release(); -} - -ObjectImage *RuntimeDyld::loadObject(ObjectBuffer *InputBuffer) { - std::unique_ptr<ObjectImage> InputImage; - sys::fs::file_magic Type = sys::fs::identify_magic(InputBuffer->getBuffer()); - - switch (Type) { - case sys::fs::file_magic::elf_relocatable: - case sys::fs::file_magic::elf_executable: - case sys::fs::file_magic::elf_shared_object: - case sys::fs::file_magic::elf_core: - InputImage.reset(RuntimeDyldELF::createObjectImage(InputBuffer)); - if (!Dyld) - Dyld = createRuntimeDyldELF(MM, ProcessAllSections).release(); - break; - case sys::fs::file_magic::macho_object: - case sys::fs::file_magic::macho_executable: - case sys::fs::file_magic::macho_fixed_virtual_memory_shared_lib: - case sys::fs::file_magic::macho_core: - case sys::fs::file_magic::macho_preload_executable: - case sys::fs::file_magic::macho_dynamically_linked_shared_lib: - case sys::fs::file_magic::macho_dynamic_linker: - case sys::fs::file_magic::macho_bundle: - case sys::fs::file_magic::macho_dynamically_linked_shared_lib_stub: - case sys::fs::file_magic::macho_dsym_companion: - InputImage.reset(RuntimeDyldMachO::createObjectImage(InputBuffer)); - if (!Dyld) +std::unique_ptr<RuntimeDyld::LoadedObjectInfo> +RuntimeDyld::loadObject(const ObjectFile &Obj) { + if (!Dyld) { + if (Obj.isELF()) + Dyld = createRuntimeDyldELF(MM, ProcessAllSections, Checker); + else if (Obj.isMachO()) Dyld = createRuntimeDyldMachO( - static_cast<Triple::ArchType>(InputImage->getArch()), - MM, ProcessAllSections).release(); - break; - case sys::fs::file_magic::unknown: - case sys::fs::file_magic::bitcode: - case sys::fs::file_magic::archive: - case sys::fs::file_magic::coff_object: - case sys::fs::file_magic::coff_import_library: - case sys::fs::file_magic::pecoff_executable: - case sys::fs::file_magic::macho_universal_binary: - case sys::fs::file_magic::windows_resource: - report_fatal_error("Incompatible object format!"); + static_cast<Triple::ArchType>(Obj.getArch()), MM, + ProcessAllSections, Checker); + else + report_fatal_error("Incompatible object format!"); } - if (!Dyld->isCompatibleFormat(InputBuffer)) + if (!Dyld->isCompatibleFile(Obj)) report_fatal_error("Incompatible object format!"); - Dyld->loadObject(InputImage.get()); - return InputImage.release(); + return Dyld->loadObject(Obj); } -void *RuntimeDyld::getSymbolAddress(StringRef Name) { +void *RuntimeDyld::getSymbolAddress(StringRef Name) const { if (!Dyld) return nullptr; return Dyld->getSymbolAddress(Name); } -uint64_t RuntimeDyld::getSymbolLoadAddress(StringRef Name) { +uint64_t RuntimeDyld::getSymbolLoadAddress(StringRef Name) const { if (!Dyld) return 0; return Dyld->getSymbolLoadAddress(Name); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index 1e63d9207f73..f3e5c77cd08c 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -7,12 +7,14 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/STLExtras.h" +#include "RuntimeDyldCheckerImpl.h" +#include "RuntimeDyldImpl.h" #include "llvm/ExecutionEngine/RuntimeDyldChecker.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler.h" #include "llvm/MC/MCInst.h" -#include "llvm/Support/StringRefMemoryObject.h" -#include "RuntimeDyldImpl.h" +#include "llvm/Support/Path.h" #include <cctype> #include <memory> @@ -22,594 +24,696 @@ using namespace llvm; namespace llvm { - // Helper class that implements the language evaluated by RuntimeDyldChecker. - class RuntimeDyldCheckerExprEval { - public: - - RuntimeDyldCheckerExprEval(const RuntimeDyldChecker &Checker, - llvm::raw_ostream &ErrStream) - : Checker(Checker), ErrStream(ErrStream) {} - - bool evaluate(StringRef Expr) const { - // Expect equality expression of the form 'LHS = RHS'. - Expr = Expr.trim(); - size_t EQIdx = Expr.find('='); - - // Evaluate LHS. - StringRef LHSExpr = Expr.substr(0, EQIdx).rtrim(); - StringRef RemainingExpr; - EvalResult LHSResult; - std::tie(LHSResult, RemainingExpr) = - evalComplexExpr(evalSimpleExpr(LHSExpr)); - if (LHSResult.hasError()) - return handleError(Expr, LHSResult); - if (RemainingExpr != "") - return handleError(Expr, unexpectedToken(RemainingExpr, LHSExpr, "")); - - // Evaluate RHS. - StringRef RHSExpr = Expr.substr(EQIdx + 1).ltrim(); - EvalResult RHSResult; - std::tie(RHSResult, RemainingExpr) = - evalComplexExpr(evalSimpleExpr(RHSExpr)); - if (RHSResult.hasError()) - return handleError(Expr, RHSResult); - if (RemainingExpr != "") - return handleError(Expr, unexpectedToken(RemainingExpr, RHSExpr, "")); - - if (LHSResult.getValue() != RHSResult.getValue()) { - ErrStream << "Expression '" << Expr << "' is false: " - << format("0x%lx", LHSResult.getValue()) << " != " - << format("0x%lx", RHSResult.getValue()) << "\n"; - return false; - } - return true; +// Helper class that implements the language evaluated by RuntimeDyldChecker. +class RuntimeDyldCheckerExprEval { +public: + RuntimeDyldCheckerExprEval(const RuntimeDyldCheckerImpl &Checker, + raw_ostream &ErrStream) + : Checker(Checker) {} + + bool evaluate(StringRef Expr) const { + // Expect equality expression of the form 'LHS = RHS'. + Expr = Expr.trim(); + size_t EQIdx = Expr.find('='); + + ParseContext OutsideLoad(false); + + // Evaluate LHS. + StringRef LHSExpr = Expr.substr(0, EQIdx).rtrim(); + StringRef RemainingExpr; + EvalResult LHSResult; + std::tie(LHSResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(LHSExpr, OutsideLoad), OutsideLoad); + if (LHSResult.hasError()) + return handleError(Expr, LHSResult); + if (RemainingExpr != "") + return handleError(Expr, unexpectedToken(RemainingExpr, LHSExpr, "")); + + // Evaluate RHS. + StringRef RHSExpr = Expr.substr(EQIdx + 1).ltrim(); + EvalResult RHSResult; + std::tie(RHSResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(RHSExpr, OutsideLoad), OutsideLoad); + if (RHSResult.hasError()) + return handleError(Expr, RHSResult); + if (RemainingExpr != "") + return handleError(Expr, unexpectedToken(RemainingExpr, RHSExpr, "")); + + if (LHSResult.getValue() != RHSResult.getValue()) { + Checker.ErrStream << "Expression '" << Expr << "' is false: " + << format("0x%" PRIx64, LHSResult.getValue()) + << " != " << format("0x%" PRIx64, RHSResult.getValue()) + << "\n"; + return false; } + return true; + } + +private: + // RuntimeDyldCheckerExprEval requires some context when parsing exprs. In + // particular, it needs to know whether a symbol is being evaluated in the + // context of a load, in which case we want the linker's local address for + // the symbol, or outside of a load, in which case we want the symbol's + // address in the remote target. + + struct ParseContext { + bool IsInsideLoad; + ParseContext(bool IsInsideLoad) : IsInsideLoad(IsInsideLoad) {} + }; + + const RuntimeDyldCheckerImpl &Checker; + + enum class BinOpToken : unsigned { + Invalid, + Add, + Sub, + BitwiseAnd, + BitwiseOr, + ShiftLeft, + ShiftRight + }; + + class EvalResult { + public: + EvalResult() : Value(0), ErrorMsg("") {} + EvalResult(uint64_t Value) : Value(Value), ErrorMsg("") {} + EvalResult(std::string ErrorMsg) : Value(0), ErrorMsg(ErrorMsg) {} + uint64_t getValue() const { return Value; } + bool hasError() const { return ErrorMsg != ""; } + const std::string &getErrorMsg() const { return ErrorMsg; } private: - const RuntimeDyldChecker &Checker; - llvm::raw_ostream &ErrStream; - - enum class BinOpToken : unsigned { Invalid, Add, Sub, BitwiseAnd, - BitwiseOr, ShiftLeft, ShiftRight }; - - class EvalResult { - public: - EvalResult() - : Value(0), ErrorMsg("") {} - EvalResult(uint64_t Value) - : Value(Value), ErrorMsg("") {} - EvalResult(std::string ErrorMsg) - : Value(0), ErrorMsg(ErrorMsg) {} - uint64_t getValue() const { return Value; } - bool hasError() const { return ErrorMsg != ""; } - const std::string& getErrorMsg() const { return ErrorMsg; } - private: - uint64_t Value; - std::string ErrorMsg; - }; - - StringRef getTokenForError(StringRef Expr) const { - if (Expr.empty()) - return ""; - - StringRef Token, Remaining; - if (isalpha(Expr[0])) - std::tie(Token, Remaining) = parseSymbol(Expr); - else if (isdigit(Expr[0])) - std::tie(Token, Remaining) = parseNumberString(Expr); - else { - unsigned TokLen = 1; - if (Expr.startswith("<<") || Expr.startswith(">>")) - TokLen = 2; - Token = Expr.substr(0, TokLen); - } - return Token; - } + uint64_t Value; + std::string ErrorMsg; + }; - EvalResult unexpectedToken(StringRef TokenStart, - StringRef SubExpr, - StringRef ErrText) const { - std::string ErrorMsg("Encountered unexpected token '"); - ErrorMsg += getTokenForError(TokenStart); - if (SubExpr != "") { - ErrorMsg += "' while parsing subexpression '"; - ErrorMsg += SubExpr; - } - ErrorMsg += "'"; - if (ErrText != "") { - ErrorMsg += " "; - ErrorMsg += ErrText; - } - return EvalResult(std::move(ErrorMsg)); + StringRef getTokenForError(StringRef Expr) const { + if (Expr.empty()) + return ""; + + StringRef Token, Remaining; + if (isalpha(Expr[0])) + std::tie(Token, Remaining) = parseSymbol(Expr); + else if (isdigit(Expr[0])) + std::tie(Token, Remaining) = parseNumberString(Expr); + else { + unsigned TokLen = 1; + if (Expr.startswith("<<") || Expr.startswith(">>")) + TokLen = 2; + Token = Expr.substr(0, TokLen); } + return Token; + } - bool handleError(StringRef Expr, const EvalResult &R) const { - assert(R.hasError() && "Not an error result."); - ErrStream << "Error evaluating expression '" << Expr << "': " - << R.getErrorMsg() << "\n"; - return false; + EvalResult unexpectedToken(StringRef TokenStart, StringRef SubExpr, + StringRef ErrText) const { + std::string ErrorMsg("Encountered unexpected token '"); + ErrorMsg += getTokenForError(TokenStart); + if (SubExpr != "") { + ErrorMsg += "' while parsing subexpression '"; + ErrorMsg += SubExpr; + } + ErrorMsg += "'"; + if (ErrText != "") { + ErrorMsg += " "; + ErrorMsg += ErrText; } + return EvalResult(std::move(ErrorMsg)); + } - std::pair<BinOpToken, StringRef> parseBinOpToken(StringRef Expr) const { - if (Expr.empty()) - return std::make_pair(BinOpToken::Invalid, ""); - - // Handle the two 2-character tokens. - if (Expr.startswith("<<")) - return std::make_pair(BinOpToken::ShiftLeft, - Expr.substr(2).ltrim()); - if (Expr.startswith(">>")) - return std::make_pair(BinOpToken::ShiftRight, - Expr.substr(2).ltrim()); - - // Handle one-character tokens. - BinOpToken Op; - switch (Expr[0]) { - default: return std::make_pair(BinOpToken::Invalid, Expr); - case '+': Op = BinOpToken::Add; break; - case '-': Op = BinOpToken::Sub; break; - case '&': Op = BinOpToken::BitwiseAnd; break; - case '|': Op = BinOpToken::BitwiseOr; break; - } + bool handleError(StringRef Expr, const EvalResult &R) const { + assert(R.hasError() && "Not an error result."); + Checker.ErrStream << "Error evaluating expression '" << Expr + << "': " << R.getErrorMsg() << "\n"; + return false; + } - return std::make_pair(Op, Expr.substr(1).ltrim()); + std::pair<BinOpToken, StringRef> parseBinOpToken(StringRef Expr) const { + if (Expr.empty()) + return std::make_pair(BinOpToken::Invalid, ""); + + // Handle the two 2-character tokens. + if (Expr.startswith("<<")) + return std::make_pair(BinOpToken::ShiftLeft, Expr.substr(2).ltrim()); + if (Expr.startswith(">>")) + return std::make_pair(BinOpToken::ShiftRight, Expr.substr(2).ltrim()); + + // Handle one-character tokens. + BinOpToken Op; + switch (Expr[0]) { + default: + return std::make_pair(BinOpToken::Invalid, Expr); + case '+': + Op = BinOpToken::Add; + break; + case '-': + Op = BinOpToken::Sub; + break; + case '&': + Op = BinOpToken::BitwiseAnd; + break; + case '|': + Op = BinOpToken::BitwiseOr; + break; } - EvalResult computeBinOpResult(BinOpToken Op, const EvalResult &LHSResult, - const EvalResult &RHSResult) const { - switch (Op) { - default: llvm_unreachable("Tried to evaluate unrecognized operation."); - case BinOpToken::Add: - return EvalResult(LHSResult.getValue() + RHSResult.getValue()); - case BinOpToken::Sub: - return EvalResult(LHSResult.getValue() - RHSResult.getValue()); - case BinOpToken::BitwiseAnd: - return EvalResult(LHSResult.getValue() & RHSResult.getValue()); - case BinOpToken::BitwiseOr: - return EvalResult(LHSResult.getValue() | RHSResult.getValue()); - case BinOpToken::ShiftLeft: - return EvalResult(LHSResult.getValue() << RHSResult.getValue()); - case BinOpToken::ShiftRight: - return EvalResult(LHSResult.getValue() >> RHSResult.getValue()); - } - } + return std::make_pair(Op, Expr.substr(1).ltrim()); + } - // Parse a symbol and return a (string, string) pair representing the symbol - // name and expression remaining to be parsed. - std::pair<StringRef, StringRef> parseSymbol(StringRef Expr) const { - size_t FirstNonSymbol = - Expr.find_first_not_of("0123456789" - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - ":_"); - return std::make_pair(Expr.substr(0, FirstNonSymbol), - Expr.substr(FirstNonSymbol).ltrim()); + EvalResult computeBinOpResult(BinOpToken Op, const EvalResult &LHSResult, + const EvalResult &RHSResult) const { + switch (Op) { + default: + llvm_unreachable("Tried to evaluate unrecognized operation."); + case BinOpToken::Add: + return EvalResult(LHSResult.getValue() + RHSResult.getValue()); + case BinOpToken::Sub: + return EvalResult(LHSResult.getValue() - RHSResult.getValue()); + case BinOpToken::BitwiseAnd: + return EvalResult(LHSResult.getValue() & RHSResult.getValue()); + case BinOpToken::BitwiseOr: + return EvalResult(LHSResult.getValue() | RHSResult.getValue()); + case BinOpToken::ShiftLeft: + return EvalResult(LHSResult.getValue() << RHSResult.getValue()); + case BinOpToken::ShiftRight: + return EvalResult(LHSResult.getValue() >> RHSResult.getValue()); } + } - // Evaluate a call to decode_operand. Decode the instruction operand at the - // given symbol and get the value of the requested operand. - // Returns an error if the instruction cannot be decoded, or the requested - // operand is not an immediate. - // On success, retuns a pair containing the value of the operand, plus - // the expression remaining to be evaluated. - std::pair<EvalResult, StringRef> evalDecodeOperand(StringRef Expr) const { - if (!Expr.startswith("(")) - return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); - StringRef RemainingExpr = Expr.substr(1).ltrim(); - StringRef Symbol; - std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); - - if (!Checker.isSymbolValid(Symbol)) - return std::make_pair(EvalResult(("Cannot decode unknown symbol '" + - Symbol + "'").str()), - ""); - - if (!RemainingExpr.startswith(",")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ','"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - EvalResult OpIdxExpr; - std::tie(OpIdxExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - if (OpIdxExpr.hasError()) - return std::make_pair(OpIdxExpr, ""); - - if (!RemainingExpr.startswith(")")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ')'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - MCInst Inst; - uint64_t Size; - if (!decodeInst(Symbol, Inst, Size)) - return std::make_pair(EvalResult(("Couldn't decode instruction at '" + - Symbol + "'").str()), - ""); - - unsigned OpIdx = OpIdxExpr.getValue(); - if (OpIdx >= Inst.getNumOperands()) { - std::string ErrMsg; - raw_string_ostream ErrMsgStream(ErrMsg); - ErrMsgStream << "Invalid operand index '" << format("%i", OpIdx) - << "' for instruction '" << Symbol - << "'. Instruction has only " - << format("%i", Inst.getNumOperands()) - << " operands.\nInstruction is:\n "; - Inst.dump_pretty(ErrMsgStream, - Checker.Disassembler->getContext().getAsmInfo(), - Checker.InstPrinter); - return std::make_pair(EvalResult(ErrMsgStream.str()), ""); - } + // Parse a symbol and return a (string, string) pair representing the symbol + // name and expression remaining to be parsed. + std::pair<StringRef, StringRef> parseSymbol(StringRef Expr) const { + size_t FirstNonSymbol = Expr.find_first_not_of("0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + ":_.$"); + return std::make_pair(Expr.substr(0, FirstNonSymbol), + Expr.substr(FirstNonSymbol).ltrim()); + } - const MCOperand &Op = Inst.getOperand(OpIdx); - if (!Op.isImm()) { - std::string ErrMsg; - raw_string_ostream ErrMsgStream(ErrMsg); - ErrMsgStream << "Operand '" << format("%i", OpIdx) - << "' of instruction '" << Symbol - << "' is not an immediate.\nInstruction is:\n "; - Inst.dump_pretty(ErrMsgStream, - Checker.Disassembler->getContext().getAsmInfo(), - Checker.InstPrinter); - - return std::make_pair(EvalResult(ErrMsgStream.str()), ""); - } + // Evaluate a call to decode_operand. Decode the instruction operand at the + // given symbol and get the value of the requested operand. + // Returns an error if the instruction cannot be decoded, or the requested + // operand is not an immediate. + // On success, retuns a pair containing the value of the operand, plus + // the expression remaining to be evaluated. + std::pair<EvalResult, StringRef> evalDecodeOperand(StringRef Expr) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + StringRef Symbol; + std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); + + if (!Checker.isSymbolValid(Symbol)) + return std::make_pair( + EvalResult(("Cannot decode unknown symbol '" + Symbol + "'").str()), + ""); - return std::make_pair(EvalResult(Op.getImm()), RemainingExpr); - } + if (!RemainingExpr.startswith(",")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - // Evaluate a call to next_pc. Decode the instruction at the given - // symbol and return the following program counter.. - // Returns an error if the instruction cannot be decoded. - // On success, returns a pair containing the next PC, plus the length of the - // expression remaining to be evaluated. - std::pair<EvalResult, StringRef> evalNextPC(StringRef Expr) const { - if (!Expr.startswith("(")) - return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); - StringRef RemainingExpr = Expr.substr(1).ltrim(); - StringRef Symbol; - std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); - - if (!Checker.isSymbolValid(Symbol)) - return std::make_pair(EvalResult(("Cannot decode unknown symbol '" - + Symbol + "'").str()), - ""); - - if (!RemainingExpr.startswith(")")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ')'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - MCInst Inst; - uint64_t Size; - if (!decodeInst(Symbol, Inst, Size)) - return std::make_pair(EvalResult(("Couldn't decode instruction at '" + - Symbol + "'").str()), - ""); - uint64_t NextPC = Checker.getSymbolAddress(Symbol) + Size; - - return std::make_pair(EvalResult(NextPC), RemainingExpr); - } + EvalResult OpIdxExpr; + std::tie(OpIdxExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + if (OpIdxExpr.hasError()) + return std::make_pair(OpIdxExpr, ""); - // Evaluate an identiefer expr, which may be a symbol, or a call to - // one of the builtin functions: get_insn_opcode or get_insn_length. - // Return the result, plus the expression remaining to be parsed. - std::pair<EvalResult, StringRef> evalIdentifierExpr(StringRef Expr) const { - StringRef Symbol; - StringRef RemainingExpr; - std::tie(Symbol, RemainingExpr) = parseSymbol(Expr); - - // Check for builtin function calls. - if (Symbol == "decode_operand") - return evalDecodeOperand(RemainingExpr); - else if (Symbol == "next_pc") - return evalNextPC(RemainingExpr); - - if (!Checker.isSymbolValid(Symbol)) { - std::string ErrMsg("No known address for symbol '"); - ErrMsg += Symbol; - ErrMsg += "'"; - if (Symbol.startswith("L")) - ErrMsg += " (this appears to be an assembler local label - " - " perhaps drop the 'L'?)"; - - return std::make_pair(EvalResult(ErrMsg), ""); - } + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - // Looks like a plain symbol reference. - return std::make_pair(EvalResult(Checker.getSymbolAddress(Symbol)), - RemainingExpr); + MCInst Inst; + uint64_t Size; + if (!decodeInst(Symbol, Inst, Size)) + return std::make_pair( + EvalResult(("Couldn't decode instruction at '" + Symbol + "'").str()), + ""); + + unsigned OpIdx = OpIdxExpr.getValue(); + if (OpIdx >= Inst.getNumOperands()) { + std::string ErrMsg; + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "Invalid operand index '" << format("%i", OpIdx) + << "' for instruction '" << Symbol + << "'. Instruction has only " + << format("%i", Inst.getNumOperands()) + << " operands.\nInstruction is:\n "; + Inst.dump_pretty(ErrMsgStream, + Checker.Disassembler->getContext().getAsmInfo(), + Checker.InstPrinter); + return std::make_pair(EvalResult(ErrMsgStream.str()), ""); } - // Parse a number (hexadecimal or decimal) and return a (string, string) - // pair representing the number and the expression remaining to be parsed. - std::pair<StringRef, StringRef> parseNumberString(StringRef Expr) const { - size_t FirstNonDigit = StringRef::npos; - if (Expr.startswith("0x")) { - FirstNonDigit = Expr.find_first_not_of("0123456789abcdefABCDEF", 2); - if (FirstNonDigit == StringRef::npos) - FirstNonDigit = Expr.size(); - } else { - FirstNonDigit = Expr.find_first_not_of("0123456789"); - if (FirstNonDigit == StringRef::npos) - FirstNonDigit = Expr.size(); - } - return std::make_pair(Expr.substr(0, FirstNonDigit), - Expr.substr(FirstNonDigit)); + const MCOperand &Op = Inst.getOperand(OpIdx); + if (!Op.isImm()) { + std::string ErrMsg; + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "Operand '" << format("%i", OpIdx) << "' of instruction '" + << Symbol << "' is not an immediate.\nInstruction is:\n "; + Inst.dump_pretty(ErrMsgStream, + Checker.Disassembler->getContext().getAsmInfo(), + Checker.InstPrinter); + + return std::make_pair(EvalResult(ErrMsgStream.str()), ""); } - // Evaluate a constant numeric expression (hexidecimal or decimal) and - // return a pair containing the result, and the expression remaining to be - // evaluated. - std::pair<EvalResult, StringRef> evalNumberExpr(StringRef Expr) const { - StringRef ValueStr; - StringRef RemainingExpr; - std::tie(ValueStr, RemainingExpr) = parseNumberString(Expr); - - if (ValueStr.empty() || !isdigit(ValueStr[0])) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected number"), - ""); - uint64_t Value; - ValueStr.getAsInteger(0, Value); - return std::make_pair(EvalResult(Value), RemainingExpr); - } + return std::make_pair(EvalResult(Op.getImm()), RemainingExpr); + } - // Evaluate an expression of the form "(<expr>)" and return a pair - // containing the result of evaluating <expr>, plus the expression - // remaining to be parsed. - std::pair<EvalResult, StringRef> evalParensExpr(StringRef Expr) const { - assert(Expr.startswith("(") && "Not a parenthesized expression"); - EvalResult SubExprResult; - StringRef RemainingExpr; - std::tie(SubExprResult, RemainingExpr) = - evalComplexExpr(evalSimpleExpr(Expr.substr(1).ltrim())); - if (SubExprResult.hasError()) - return std::make_pair(SubExprResult, ""); - if (!RemainingExpr.startswith(")")) - return std::make_pair(unexpectedToken(RemainingExpr, Expr, - "expected ')'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - return std::make_pair(SubExprResult, RemainingExpr); - } + // Evaluate a call to next_pc. + // Decode the instruction at the given symbol and return the following program + // counter. + // Returns an error if the instruction cannot be decoded. + // On success, returns a pair containing the next PC, plus of the + // expression remaining to be evaluated. + std::pair<EvalResult, StringRef> evalNextPC(StringRef Expr, + ParseContext PCtx) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + StringRef Symbol; + std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); + + if (!Checker.isSymbolValid(Symbol)) + return std::make_pair( + EvalResult(("Cannot decode unknown symbol '" + Symbol + "'").str()), + ""); - // Evaluate an expression in one of the following forms: - // *{<number>}<symbol> - // *{<number>}(<symbol> + <number>) - // *{<number>}(<symbol> - <number>) - // Return a pair containing the result, plus the expression remaining to be - // parsed. - std::pair<EvalResult, StringRef> evalLoadExpr(StringRef Expr) const { - assert(Expr.startswith("*") && "Not a load expression"); - StringRef RemainingExpr = Expr.substr(1).ltrim(); - // Parse read size. - if (!RemainingExpr.startswith("{")) - return std::make_pair(EvalResult("Expected '{' following '*'."), ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - EvalResult ReadSizeExpr; - std::tie(ReadSizeExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - if (ReadSizeExpr.hasError()) - return std::make_pair(ReadSizeExpr, RemainingExpr); - uint64_t ReadSize = ReadSizeExpr.getValue(); - if (ReadSize < 1 || ReadSize > 8) - return std::make_pair(EvalResult("Invalid size for dereference."), ""); - if (!RemainingExpr.startswith("}")) - return std::make_pair(EvalResult("Missing '}' for dereference."), ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - // Check for '(symbol +/- constant)' form. - bool SymbolPlusConstant = false; - if (RemainingExpr.startswith("(")) { - SymbolPlusConstant = true; - RemainingExpr = RemainingExpr.substr(1).ltrim(); - } + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + MCInst Inst; + uint64_t InstSize; + if (!decodeInst(Symbol, Inst, InstSize)) + return std::make_pair( + EvalResult(("Couldn't decode instruction at '" + Symbol + "'").str()), + ""); + + uint64_t SymbolAddr = PCtx.IsInsideLoad + ? Checker.getSymbolLinkerAddr(Symbol) + : Checker.getSymbolRemoteAddr(Symbol); + uint64_t NextPC = SymbolAddr + InstSize; + + return std::make_pair(EvalResult(NextPC), RemainingExpr); + } + + // Evaluate a call to stub_addr. + // Look up and return the address of the stub for the given + // (<file name>, <section name>, <symbol name>) tuple. + // On success, returns a pair containing the stub address, plus the expression + // remaining to be evaluated. + std::pair<EvalResult, StringRef> evalStubAddr(StringRef Expr, + ParseContext PCtx) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + + // Handle file-name specially, as it may contain characters that aren't + // legal for symbols. + StringRef FileName; + size_t ComaIdx = RemainingExpr.find(','); + FileName = RemainingExpr.substr(0, ComaIdx).rtrim(); + RemainingExpr = RemainingExpr.substr(ComaIdx).ltrim(); + + if (!RemainingExpr.startswith(",")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - // Read symbol. - StringRef Symbol; - std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); + StringRef SectionName; + std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr); - if (!Checker.isSymbolValid(Symbol)) - return std::make_pair(EvalResult(("Cannot dereference unknown symbol '" - + Symbol + "'").str()), - ""); + if (!RemainingExpr.startswith(",")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - // Set up defaut offset. - int64_t Offset = 0; + StringRef Symbol; + std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); - // Handle "+/- constant)" portion if necessary. - if (SymbolPlusConstant) { - char OpChar = RemainingExpr[0]; - if (OpChar != '+' && OpChar != '-') - return std::make_pair(EvalResult("Invalid operator in load address."), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - EvalResult OffsetExpr; - std::tie(OffsetExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + uint64_t StubAddr; + std::string ErrorMsg = ""; + std::tie(StubAddr, ErrorMsg) = Checker.getStubAddrFor( + FileName, SectionName, Symbol, PCtx.IsInsideLoad); - Offset = (OpChar == '+') ? - OffsetExpr.getValue() : -1 * OffsetExpr.getValue(); + if (ErrorMsg != "") + return std::make_pair(EvalResult(ErrorMsg), ""); - if (!RemainingExpr.startswith(")")) - return std::make_pair(EvalResult("Missing ')' in load address."), - ""); + return std::make_pair(EvalResult(StubAddr), RemainingExpr); + } - RemainingExpr = RemainingExpr.substr(1).ltrim(); - } + std::pair<EvalResult, StringRef> evalSectionAddr(StringRef Expr, + ParseContext PCtx) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + // Handle file-name specially, as it may contain characters that aren't + // legal for symbols. + StringRef FileName; + size_t ComaIdx = RemainingExpr.find(','); + FileName = RemainingExpr.substr(0, ComaIdx).rtrim(); + RemainingExpr = RemainingExpr.substr(ComaIdx).ltrim(); + + if (!RemainingExpr.startswith(",")) return std::make_pair( - EvalResult(Checker.readMemoryAtSymbol(Symbol, Offset, ReadSize)), - RemainingExpr); + unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + StringRef SectionName; + std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr); + + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + uint64_t StubAddr; + std::string ErrorMsg = ""; + std::tie(StubAddr, ErrorMsg) = Checker.getSectionAddr( + FileName, SectionName, PCtx.IsInsideLoad); + + if (ErrorMsg != "") + return std::make_pair(EvalResult(ErrorMsg), ""); + + return std::make_pair(EvalResult(StubAddr), RemainingExpr); + } + + // Evaluate an identiefer expr, which may be a symbol, or a call to + // one of the builtin functions: get_insn_opcode or get_insn_length. + // Return the result, plus the expression remaining to be parsed. + std::pair<EvalResult, StringRef> evalIdentifierExpr(StringRef Expr, + ParseContext PCtx) const { + StringRef Symbol; + StringRef RemainingExpr; + std::tie(Symbol, RemainingExpr) = parseSymbol(Expr); + + // Check for builtin function calls. + if (Symbol == "decode_operand") + return evalDecodeOperand(RemainingExpr); + else if (Symbol == "next_pc") + return evalNextPC(RemainingExpr, PCtx); + else if (Symbol == "stub_addr") + return evalStubAddr(RemainingExpr, PCtx); + else if (Symbol == "section_addr") + return evalSectionAddr(RemainingExpr, PCtx); + + if (!Checker.isSymbolValid(Symbol)) { + std::string ErrMsg("No known address for symbol '"); + ErrMsg += Symbol; + ErrMsg += "'"; + if (Symbol.startswith("L")) + ErrMsg += " (this appears to be an assembler local label - " + " perhaps drop the 'L'?)"; + + return std::make_pair(EvalResult(ErrMsg), ""); } - // Evaluate a "simple" expression. This is any expression that _isn't_ an - // un-parenthesized binary expression. - // - // "Simple" expressions can be optionally bit-sliced. See evalSlicedExpr. - // - // Returns a pair containing the result of the evaluation, plus the - // expression remaining to be parsed. - std::pair<EvalResult, StringRef> evalSimpleExpr(StringRef Expr) const { - EvalResult SubExprResult; - StringRef RemainingExpr; - - if (Expr.empty()) - return std::make_pair(EvalResult("Unexpected end of expression"), ""); - - if (Expr[0] == '(') - std::tie(SubExprResult, RemainingExpr) = evalParensExpr(Expr); - else if (Expr[0] == '*') - std::tie(SubExprResult, RemainingExpr) = evalLoadExpr(Expr); - else if (isalpha(Expr[0])) - std::tie(SubExprResult, RemainingExpr) = evalIdentifierExpr(Expr); - else if (isdigit(Expr[0])) - std::tie(SubExprResult, RemainingExpr) = evalNumberExpr(Expr); - - if (SubExprResult.hasError()) - return std::make_pair(SubExprResult, RemainingExpr); - - // Evaluate bit-slice if present. - if (RemainingExpr.startswith("[")) - std::tie(SubExprResult, RemainingExpr) = - evalSliceExpr(std::make_pair(SubExprResult, RemainingExpr)); + // The value for the symbol depends on the context we're evaluating in: + // Inside a load this is the address in the linker's memory, outside a + // load it's the address in the target processes memory. + uint64_t Value = PCtx.IsInsideLoad ? Checker.getSymbolLinkerAddr(Symbol) + : Checker.getSymbolRemoteAddr(Symbol); - return std::make_pair(SubExprResult, RemainingExpr); + // Looks like a plain symbol reference. + return std::make_pair(EvalResult(Value), RemainingExpr); + } + + // Parse a number (hexadecimal or decimal) and return a (string, string) + // pair representing the number and the expression remaining to be parsed. + std::pair<StringRef, StringRef> parseNumberString(StringRef Expr) const { + size_t FirstNonDigit = StringRef::npos; + if (Expr.startswith("0x")) { + FirstNonDigit = Expr.find_first_not_of("0123456789abcdefABCDEF", 2); + if (FirstNonDigit == StringRef::npos) + FirstNonDigit = Expr.size(); + } else { + FirstNonDigit = Expr.find_first_not_of("0123456789"); + if (FirstNonDigit == StringRef::npos) + FirstNonDigit = Expr.size(); } + return std::make_pair(Expr.substr(0, FirstNonDigit), + Expr.substr(FirstNonDigit)); + } + + // Evaluate a constant numeric expression (hexidecimal or decimal) and + // return a pair containing the result, and the expression remaining to be + // evaluated. + std::pair<EvalResult, StringRef> evalNumberExpr(StringRef Expr) const { + StringRef ValueStr; + StringRef RemainingExpr; + std::tie(ValueStr, RemainingExpr) = parseNumberString(Expr); + + if (ValueStr.empty() || !isdigit(ValueStr[0])) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected number"), ""); + uint64_t Value; + ValueStr.getAsInteger(0, Value); + return std::make_pair(EvalResult(Value), RemainingExpr); + } + + // Evaluate an expression of the form "(<expr>)" and return a pair + // containing the result of evaluating <expr>, plus the expression + // remaining to be parsed. + std::pair<EvalResult, StringRef> evalParensExpr(StringRef Expr, + ParseContext PCtx) const { + assert(Expr.startswith("(") && "Not a parenthesized expression"); + EvalResult SubExprResult; + StringRef RemainingExpr; + std::tie(SubExprResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(Expr.substr(1).ltrim(), PCtx), PCtx); + if (SubExprResult.hasError()) + return std::make_pair(SubExprResult, ""); + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + return std::make_pair(SubExprResult, RemainingExpr); + } + + // Evaluate an expression in one of the following forms: + // *{<number>}<expr> + // Return a pair containing the result, plus the expression remaining to be + // parsed. + std::pair<EvalResult, StringRef> evalLoadExpr(StringRef Expr) const { + assert(Expr.startswith("*") && "Not a load expression"); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + + // Parse read size. + if (!RemainingExpr.startswith("{")) + return std::make_pair(EvalResult("Expected '{' following '*'."), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + EvalResult ReadSizeExpr; + std::tie(ReadSizeExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + if (ReadSizeExpr.hasError()) + return std::make_pair(ReadSizeExpr, RemainingExpr); + uint64_t ReadSize = ReadSizeExpr.getValue(); + if (ReadSize < 1 || ReadSize > 8) + return std::make_pair(EvalResult("Invalid size for dereference."), ""); + if (!RemainingExpr.startswith("}")) + return std::make_pair(EvalResult("Missing '}' for dereference."), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + // Evaluate the expression representing the load address. + ParseContext LoadCtx(true); + EvalResult LoadAddrExprResult; + std::tie(LoadAddrExprResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(RemainingExpr, LoadCtx), LoadCtx); + + if (LoadAddrExprResult.hasError()) + return std::make_pair(LoadAddrExprResult, ""); + + uint64_t LoadAddr = LoadAddrExprResult.getValue(); + + return std::make_pair( + EvalResult(Checker.readMemoryAtAddr(LoadAddr, ReadSize)), + RemainingExpr); + } + + // Evaluate a "simple" expression. This is any expression that _isn't_ an + // un-parenthesized binary expression. + // + // "Simple" expressions can be optionally bit-sliced. See evalSlicedExpr. + // + // Returns a pair containing the result of the evaluation, plus the + // expression remaining to be parsed. + std::pair<EvalResult, StringRef> evalSimpleExpr(StringRef Expr, + ParseContext PCtx) const { + EvalResult SubExprResult; + StringRef RemainingExpr; + + if (Expr.empty()) + return std::make_pair(EvalResult("Unexpected end of expression"), ""); + + if (Expr[0] == '(') + std::tie(SubExprResult, RemainingExpr) = evalParensExpr(Expr, PCtx); + else if (Expr[0] == '*') + std::tie(SubExprResult, RemainingExpr) = evalLoadExpr(Expr); + else if (isalpha(Expr[0]) || Expr[0] == '_') + std::tie(SubExprResult, RemainingExpr) = evalIdentifierExpr(Expr, PCtx); + else if (isdigit(Expr[0])) + std::tie(SubExprResult, RemainingExpr) = evalNumberExpr(Expr); + else + return std::make_pair( + unexpectedToken(Expr, Expr, + "expected '(', '*', identifier, or number"), ""); + + if (SubExprResult.hasError()) + return std::make_pair(SubExprResult, RemainingExpr); + + // Evaluate bit-slice if present. + if (RemainingExpr.startswith("[")) + std::tie(SubExprResult, RemainingExpr) = + evalSliceExpr(std::make_pair(SubExprResult, RemainingExpr)); - // Evaluate a bit-slice of an expression. - // A bit-slice has the form "<expr>[high:low]". The result of evaluating a - // slice is the bits between high and low (inclusive) in the original - // expression, right shifted so that the "low" bit is in position 0 in the + return std::make_pair(SubExprResult, RemainingExpr); + } + + // Evaluate a bit-slice of an expression. + // A bit-slice has the form "<expr>[high:low]". The result of evaluating a + // slice is the bits between high and low (inclusive) in the original + // expression, right shifted so that the "low" bit is in position 0 in the + // result. + // Returns a pair containing the result of the slice operation, plus the + // expression remaining to be parsed. + std::pair<EvalResult, StringRef> + evalSliceExpr(std::pair<EvalResult, StringRef> Ctx) const { + EvalResult SubExprResult; + StringRef RemainingExpr; + std::tie(SubExprResult, RemainingExpr) = Ctx; + + assert(RemainingExpr.startswith("[") && "Not a slice expr."); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + EvalResult HighBitExpr; + std::tie(HighBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + + if (HighBitExpr.hasError()) + return std::make_pair(HighBitExpr, RemainingExpr); + + if (!RemainingExpr.startswith(":")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ':'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + EvalResult LowBitExpr; + std::tie(LowBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + + if (LowBitExpr.hasError()) + return std::make_pair(LowBitExpr, RemainingExpr); + + if (!RemainingExpr.startswith("]")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ']'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + unsigned HighBit = HighBitExpr.getValue(); + unsigned LowBit = LowBitExpr.getValue(); + uint64_t Mask = ((uint64_t)1 << (HighBit - LowBit + 1)) - 1; + uint64_t SlicedValue = (SubExprResult.getValue() >> LowBit) & Mask; + return std::make_pair(EvalResult(SlicedValue), RemainingExpr); + } + + // Evaluate a "complex" expression. + // Takes an already evaluated subexpression and checks for the presence of a + // binary operator, computing the result of the binary operation if one is + // found. Used to make arithmetic expressions left-associative. + // Returns a pair containing the ultimate result of evaluating the + // expression, plus the expression remaining to be evaluated. + std::pair<EvalResult, StringRef> + evalComplexExpr(std::pair<EvalResult, StringRef> LHSAndRemaining, + ParseContext PCtx) const { + EvalResult LHSResult; + StringRef RemainingExpr; + std::tie(LHSResult, RemainingExpr) = LHSAndRemaining; + + // If there was an error, or there's nothing left to evaluate, return the // result. - // Returns a pair containing the result of the slice operation, plus the - // expression remaining to be parsed. - std::pair<EvalResult, StringRef> evalSliceExpr( - std::pair<EvalResult, StringRef> Ctx) const{ - EvalResult SubExprResult; - StringRef RemainingExpr; - std::tie(SubExprResult, RemainingExpr) = Ctx; - - assert(RemainingExpr.startswith("[") && "Not a slice expr."); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - EvalResult HighBitExpr; - std::tie(HighBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - - if (HighBitExpr.hasError()) - return std::make_pair(HighBitExpr, RemainingExpr); - - if (!RemainingExpr.startswith(":")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ':'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - EvalResult LowBitExpr; - std::tie(LowBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - - if (LowBitExpr.hasError()) - return std::make_pair(LowBitExpr, RemainingExpr); - - if (!RemainingExpr.startswith("]")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ']'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - unsigned HighBit = HighBitExpr.getValue(); - unsigned LowBit = LowBitExpr.getValue(); - uint64_t Mask = ((uint64_t)1 << (HighBit - LowBit + 1)) - 1; - uint64_t SlicedValue = (SubExprResult.getValue() >> LowBit) & Mask; - return std::make_pair(EvalResult(SlicedValue), RemainingExpr); - } + if (LHSResult.hasError() || RemainingExpr == "") + return std::make_pair(LHSResult, RemainingExpr); - // Evaluate a "complex" expression. - // Takes an already evaluated subexpression and checks for the presence of a - // binary operator, computing the result of the binary operation if one is - // found. Used to make arithmetic expressions left-associative. - // Returns a pair containing the ultimate result of evaluating the - // expression, plus the expression remaining to be evaluated. - std::pair<EvalResult, StringRef> evalComplexExpr( - std::pair<EvalResult, StringRef> Ctx) const { - EvalResult LHSResult; - StringRef RemainingExpr; - std::tie(LHSResult, RemainingExpr) = Ctx; - - // If there was an error, or there's nothing left to evaluate, return the - // result. - if (LHSResult.hasError() || RemainingExpr == "") - return std::make_pair(LHSResult, RemainingExpr); - - // Otherwise check if this is a binary expressioan. - BinOpToken BinOp; - std::tie(BinOp, RemainingExpr) = parseBinOpToken(RemainingExpr); - - // If this isn't a recognized expression just return. - if (BinOp == BinOpToken::Invalid) - return std::make_pair(LHSResult, RemainingExpr); - - // This is a recognized bin-op. Evaluate the RHS, then evaluate the binop. - EvalResult RHSResult; - std::tie(RHSResult, RemainingExpr) = evalSimpleExpr(RemainingExpr); - - // If there was an error evaluating the RHS, return it. - if (RHSResult.hasError()) - return std::make_pair(RHSResult, RemainingExpr); - - // This is a binary expression - evaluate and try to continue as a - // complex expr. - EvalResult ThisResult(computeBinOpResult(BinOp, LHSResult, RHSResult)); - - return evalComplexExpr(std::make_pair(ThisResult, RemainingExpr)); - } + // Otherwise check if this is a binary expressioan. + BinOpToken BinOp; + std::tie(BinOp, RemainingExpr) = parseBinOpToken(RemainingExpr); - bool decodeInst(StringRef Symbol, MCInst &Inst, uint64_t &Size) const { - MCDisassembler *Dis = Checker.Disassembler; - StringRef SectionMem = Checker.getSubsectionStartingAt(Symbol); - StringRefMemoryObject SectionBytes(SectionMem, 0); + // If this isn't a recognized expression just return. + if (BinOp == BinOpToken::Invalid) + return std::make_pair(LHSResult, RemainingExpr); - MCDisassembler::DecodeStatus S = - Dis->getInstruction(Inst, Size, SectionBytes, 0, nulls(), nulls()); + // This is a recognized bin-op. Evaluate the RHS, then evaluate the binop. + EvalResult RHSResult; + std::tie(RHSResult, RemainingExpr) = evalSimpleExpr(RemainingExpr, PCtx); - return (S == MCDisassembler::Success); - } + // If there was an error evaluating the RHS, return it. + if (RHSResult.hasError()) + return std::make_pair(RHSResult, RemainingExpr); - }; + // This is a binary expression - evaluate and try to continue as a + // complex expr. + EvalResult ThisResult(computeBinOpResult(BinOp, LHSResult, RHSResult)); + + return evalComplexExpr(std::make_pair(ThisResult, RemainingExpr), PCtx); + } + + bool decodeInst(StringRef Symbol, MCInst &Inst, uint64_t &Size) const { + MCDisassembler *Dis = Checker.Disassembler; + StringRef SectionMem = Checker.getSubsectionStartingAt(Symbol); + ArrayRef<uint8_t> SectionBytes( + reinterpret_cast<const uint8_t *>(SectionMem.data()), + SectionMem.size()); + + MCDisassembler::DecodeStatus S = + Dis->getInstruction(Inst, Size, SectionBytes, 0, nulls(), nulls()); + + return (S == MCDisassembler::Success); + } +}; +} +RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, + MCDisassembler *Disassembler, + MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : RTDyld(RTDyld), Disassembler(Disassembler), InstPrinter(InstPrinter), + ErrStream(ErrStream) { + RTDyld.Checker = this; } -bool RuntimeDyldChecker::check(StringRef CheckExpr) const { +bool RuntimeDyldCheckerImpl::check(StringRef CheckExpr) const { CheckExpr = CheckExpr.trim(); - DEBUG(llvm::dbgs() << "RuntimeDyldChecker: Checking '" << CheckExpr - << "'...\n"); + DEBUG(dbgs() << "RuntimeDyldChecker: Checking '" << CheckExpr << "'...\n"); RuntimeDyldCheckerExprEval P(*this, ErrStream); bool Result = P.evaluate(CheckExpr); (void)Result; - DEBUG(llvm::dbgs() << "RuntimeDyldChecker: '" << CheckExpr << "' " - << (Result ? "passed" : "FAILED") << ".\n"); + DEBUG(dbgs() << "RuntimeDyldChecker: '" << CheckExpr << "' " + << (Result ? "passed" : "FAILED") << ".\n"); return Result; } -bool RuntimeDyldChecker::checkAllRulesInBuffer(StringRef RulePrefix, - MemoryBuffer* MemBuf) const { +bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix, + MemoryBuffer *MemBuf) const { bool DidAllTestsPass = true; unsigned NumRules = 0; const char *LineStart = MemBuf->getBufferStart(); // Eat whitespace. - while (LineStart != MemBuf->getBufferEnd() && - std::isspace(*LineStart)) + while (LineStart != MemBuf->getBufferEnd() && std::isspace(*LineStart)) ++LineStart; while (LineStart != MemBuf->getBufferEnd() && *LineStart != '\0') { const char *LineEnd = LineStart; - while (LineEnd != MemBuf->getBufferEnd() && - *LineEnd != '\r' && *LineEnd != '\n') + while (LineEnd != MemBuf->getBufferEnd() && *LineEnd != '\r' && + *LineEnd != '\n') ++LineEnd; StringRef Line(LineStart, LineEnd - LineStart); @@ -620,37 +724,212 @@ bool RuntimeDyldChecker::checkAllRulesInBuffer(StringRef RulePrefix, // Eat whitespace. LineStart = LineEnd; - while (LineStart != MemBuf->getBufferEnd() && - std::isspace(*LineStart)) + while (LineStart != MemBuf->getBufferEnd() && std::isspace(*LineStart)) ++LineStart; } return DidAllTestsPass && (NumRules != 0); } -bool RuntimeDyldChecker::isSymbolValid(StringRef Symbol) const { - return RTDyld.getSymbolAddress(Symbol) != nullptr; +bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const { + return getRTDyld().getSymbolAddress(Symbol) != nullptr; } -uint64_t RuntimeDyldChecker::getSymbolAddress(StringRef Symbol) const { - return RTDyld.getAnySymbolRemoteAddress(Symbol); +uint64_t RuntimeDyldCheckerImpl::getSymbolLinkerAddr(StringRef Symbol) const { + return static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(getRTDyld().getSymbolAddress(Symbol))); } -uint64_t RuntimeDyldChecker::readMemoryAtSymbol(StringRef Symbol, - int64_t Offset, - unsigned Size) const { - uint8_t *Src = RTDyld.getSymbolAddress(Symbol); - uint64_t Result = 0; - memcpy(&Result, Src + Offset, Size); - return Result; +uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { + if (uint64_t InternalSymbolAddr = getRTDyld().getSymbolLoadAddress(Symbol)) + return InternalSymbolAddr; + return getRTDyld().MemMgr->getSymbolAddress(Symbol); +} + +uint64_t RuntimeDyldCheckerImpl::readMemoryAtAddr(uint64_t SrcAddr, + unsigned Size) const { + uintptr_t PtrSizedAddr = static_cast<uintptr_t>(SrcAddr); + assert(PtrSizedAddr == SrcAddr && "Linker memory pointer out-of-range."); + uint8_t *Src = reinterpret_cast<uint8_t*>(PtrSizedAddr); + return getRTDyld().readBytesUnaligned(Src, Size); +} + + +std::pair<const RuntimeDyldCheckerImpl::SectionAddressInfo*, std::string> +RuntimeDyldCheckerImpl::findSectionAddrInfo(StringRef FileName, + StringRef SectionName) const { + + auto SectionMapItr = Stubs.find(FileName); + if (SectionMapItr == Stubs.end()) { + std::string ErrorMsg = "File '"; + ErrorMsg += FileName; + ErrorMsg += "' not found. "; + if (Stubs.empty()) + ErrorMsg += "No stubs registered."; + else { + ErrorMsg += "Available files are:"; + for (const auto& StubEntry : Stubs) { + ErrorMsg += " '"; + ErrorMsg += StubEntry.first; + ErrorMsg += "'"; + } + } + ErrorMsg += "\n"; + return std::make_pair(nullptr, ErrorMsg); + } + + auto SectionInfoItr = SectionMapItr->second.find(SectionName); + if (SectionInfoItr == SectionMapItr->second.end()) + return std::make_pair(nullptr, + ("Section '" + SectionName + "' not found in file '" + + FileName + "'\n").str()); + + return std::make_pair(&SectionInfoItr->second, std::string("")); +} + +std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getSectionAddr( + StringRef FileName, StringRef SectionName, bool IsInsideLoad) const { + + const SectionAddressInfo *SectionInfo = nullptr; + { + std::string ErrorMsg; + std::tie(SectionInfo, ErrorMsg) = + findSectionAddrInfo(FileName, SectionName); + if (ErrorMsg != "") + return std::make_pair(0, ErrorMsg); + } + + unsigned SectionID = SectionInfo->SectionID; + uint64_t Addr; + if (IsInsideLoad) + Addr = + static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(getRTDyld().Sections[SectionID].Address)); + else + Addr = getRTDyld().Sections[SectionID].LoadAddress; + + return std::make_pair(Addr, std::string("")); +} + +std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getStubAddrFor( + StringRef FileName, StringRef SectionName, StringRef SymbolName, + bool IsInsideLoad) const { + + const SectionAddressInfo *SectionInfo = nullptr; + { + std::string ErrorMsg; + std::tie(SectionInfo, ErrorMsg) = + findSectionAddrInfo(FileName, SectionName); + if (ErrorMsg != "") + return std::make_pair(0, ErrorMsg); + } + + unsigned SectionID = SectionInfo->SectionID; + const StubOffsetsMap &SymbolStubs = SectionInfo->StubOffsets; + auto StubOffsetItr = SymbolStubs.find(SymbolName); + if (StubOffsetItr == SymbolStubs.end()) + return std::make_pair(0, + ("Stub for symbol '" + SymbolName + "' not found. " + "If '" + SymbolName + "' is an internal symbol this " + "may indicate that the stub target offset is being " + "computed incorrectly.\n").str()); + + uint64_t StubOffset = StubOffsetItr->second; + + uint64_t Addr; + if (IsInsideLoad) { + uintptr_t SectionBase = + reinterpret_cast<uintptr_t>(getRTDyld().Sections[SectionID].Address); + Addr = static_cast<uint64_t>(SectionBase) + StubOffset; + } else { + uint64_t SectionBase = getRTDyld().Sections[SectionID].LoadAddress; + Addr = SectionBase + StubOffset; + } + + return std::make_pair(Addr, std::string("")); } -StringRef RuntimeDyldChecker::getSubsectionStartingAt(StringRef Name) const { +StringRef +RuntimeDyldCheckerImpl::getSubsectionStartingAt(StringRef Name) const { RuntimeDyldImpl::SymbolTableMap::const_iterator pos = - RTDyld.GlobalSymbolTable.find(Name); - if (pos == RTDyld.GlobalSymbolTable.end()) + getRTDyld().GlobalSymbolTable.find(Name); + if (pos == getRTDyld().GlobalSymbolTable.end()) return StringRef(); RuntimeDyldImpl::SymbolLoc Loc = pos->second; - uint8_t *SectionAddr = RTDyld.getSectionAddress(Loc.first); - return StringRef(reinterpret_cast<const char*>(SectionAddr) + Loc.second, - RTDyld.Sections[Loc.first].Size - Loc.second); + uint8_t *SectionAddr = getRTDyld().getSectionAddress(Loc.first); + return StringRef(reinterpret_cast<const char *>(SectionAddr) + Loc.second, + getRTDyld().Sections[Loc.first].Size - Loc.second); +} + +void RuntimeDyldCheckerImpl::registerSection( + StringRef FilePath, unsigned SectionID) { + StringRef FileName = sys::path::filename(FilePath); + const SectionEntry &Section = getRTDyld().Sections[SectionID]; + StringRef SectionName = Section.Name; + + Stubs[FileName][SectionName].SectionID = SectionID; +} + +void RuntimeDyldCheckerImpl::registerStubMap( + StringRef FilePath, unsigned SectionID, + const RuntimeDyldImpl::StubMap &RTDyldStubs) { + StringRef FileName = sys::path::filename(FilePath); + const SectionEntry &Section = getRTDyld().Sections[SectionID]; + StringRef SectionName = Section.Name; + + Stubs[FileName][SectionName].SectionID = SectionID; + + for (auto &StubMapEntry : RTDyldStubs) { + std::string SymbolName = ""; + + if (StubMapEntry.first.SymbolName) + SymbolName = StubMapEntry.first.SymbolName; + else { + // If this is a (Section, Offset) pair, do a reverse lookup in the + // global symbol table to find the name. + for (auto &GSTEntry : getRTDyld().GlobalSymbolTable) { + if (GSTEntry.second.first == StubMapEntry.first.SectionID && + GSTEntry.second.second == + static_cast<uint64_t>(StubMapEntry.first.Offset)) { + SymbolName = GSTEntry.first(); + break; + } + } + } + + if (SymbolName != "") + Stubs[FileName][SectionName].StubOffsets[SymbolName] = + StubMapEntry.second; + } +} + +RuntimeDyldChecker::RuntimeDyldChecker(RuntimeDyld &RTDyld, + MCDisassembler *Disassembler, + MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : Impl(make_unique<RuntimeDyldCheckerImpl>(RTDyld, Disassembler, + InstPrinter, ErrStream)) {} + +RuntimeDyldChecker::~RuntimeDyldChecker() {} + +RuntimeDyld& RuntimeDyldChecker::getRTDyld() { + return Impl->RTDyld; +} + +const RuntimeDyld& RuntimeDyldChecker::getRTDyld() const { + return Impl->RTDyld; +} + +bool RuntimeDyldChecker::check(StringRef CheckExpr) const { + return Impl->check(CheckExpr); +} + +bool RuntimeDyldChecker::checkAllRulesInBuffer(StringRef RulePrefix, + MemoryBuffer *MemBuf) const { + return Impl->checkAllRulesInBuffer(RulePrefix, MemBuf); +} + +std::pair<uint64_t, std::string> +RuntimeDyldChecker::getSectionAddr(StringRef FileName, StringRef SectionName, + bool LinkerAddress) { + return Impl->getSectionAddr(FileName, SectionName, LinkerAddress); } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h new file mode 100644 index 000000000000..de20c1ec6603 --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -0,0 +1,76 @@ +//===-- RuntimeDyldCheckerImpl.h -- RuntimeDyld test framework --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDCHECKERIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDCHECKERIMPL_H + +#include "RuntimeDyldImpl.h" +#include <set> + +namespace llvm { + +class RuntimeDyldCheckerImpl { + friend class RuntimeDyldChecker; + friend class RuntimeDyldImpl; + friend class RuntimeDyldCheckerExprEval; + +public: + RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, MCDisassembler *Disassembler, + MCInstPrinter *InstPrinter, + llvm::raw_ostream &ErrStream); + + bool check(StringRef CheckExpr) const; + bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const; + +private: + + // StubMap typedefs. + typedef std::map<std::string, uint64_t> StubOffsetsMap; + struct SectionAddressInfo { + uint64_t SectionID; + StubOffsetsMap StubOffsets; + }; + typedef std::map<std::string, SectionAddressInfo> SectionMap; + typedef std::map<std::string, SectionMap> StubMap; + + RuntimeDyldImpl &getRTDyld() const { return *RTDyld.Dyld; } + + bool isSymbolValid(StringRef Symbol) const; + uint64_t getSymbolLinkerAddr(StringRef Symbol) const; + uint64_t getSymbolRemoteAddr(StringRef Symbol) const; + uint64_t readMemoryAtAddr(uint64_t Addr, unsigned Size) const; + + std::pair<const SectionAddressInfo*, std::string> findSectionAddrInfo( + StringRef FileName, + StringRef SectionName) const; + + std::pair<uint64_t, std::string> getSectionAddr(StringRef FileName, + StringRef SectionName, + bool IsInsideLoad) const; + + std::pair<uint64_t, std::string> getStubAddrFor(StringRef FileName, + StringRef SectionName, + StringRef Symbol, + bool IsInsideLoad) const; + StringRef getSubsectionStartingAt(StringRef Name) const; + + void registerSection(StringRef FilePath, unsigned SectionID); + void registerStubMap(StringRef FilePath, unsigned SectionID, + const RuntimeDyldImpl::StubMap &RTDyldStubs); + + RuntimeDyld &RTDyld; + MCDisassembler *Disassembler; + MCInstPrinter *InstPrinter; + llvm::raw_ostream &ErrStream; + + StubMap Stubs; +}; +} + +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 728138ed8c16..2664a10ece5f 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -12,26 +12,23 @@ //===----------------------------------------------------------------------===// #include "RuntimeDyldELF.h" -#include "JITRegistrar.h" -#include "ObjectImageCommon.h" #include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" -#include "llvm/ExecutionEngine/ObjectBuffer.h" -#include "llvm/ExecutionEngine/ObjectImage.h" +#include "llvm/MC/MCStreamer.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/ELF.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TargetRegistry.h" using namespace llvm; using namespace llvm::object; #define DEBUG_TYPE "dyld" -namespace { - static inline std::error_code check(std::error_code Err) { if (Err) { report_fatal_error(Err.message()); @@ -39,6 +36,8 @@ static inline std::error_code check(std::error_code Err) { return Err; } +namespace { + template <class ELFT> class DyldELFObject : public ELFObjectFile<ELFT> { LLVM_ELF_IMPORT_TYPES_ELFT(ELFT) @@ -51,16 +50,12 @@ template <class ELFT> class DyldELFObject : public ELFObjectFile<ELFT> { typedef typename ELFDataTypeTypedefHelper<ELFT>::value_type addr_type; - std::unique_ptr<ObjectFile> UnderlyingFile; - public: - DyldELFObject(std::unique_ptr<ObjectFile> UnderlyingFile, - std::unique_ptr<MemoryBuffer> Wrapper, std::error_code &ec); - - DyldELFObject(std::unique_ptr<MemoryBuffer> Wrapper, std::error_code &ec); + DyldELFObject(MemoryBufferRef Wrapper, std::error_code &ec); void updateSectionAddress(const SectionRef &Sec, uint64_t Addr); - void updateSymbolAddress(const SymbolRef &Sym, uint64_t Addr); + + void updateSymbolAddress(const SymbolRef &SymRef, uint64_t Addr); // Methods for type inquiry through isa, cast and dyn_cast static inline bool classof(const Binary *v) { @@ -70,57 +65,17 @@ public: static inline bool classof(const ELFObjectFile<ELFT> *v) { return v->isDyldType(); } -}; - -template <class ELFT> class ELFObjectImage : public ObjectImageCommon { - bool Registered; - -public: - ELFObjectImage(ObjectBuffer *Input, std::unique_ptr<DyldELFObject<ELFT>> Obj) - : ObjectImageCommon(Input, std::move(Obj)), Registered(false) {} - - virtual ~ELFObjectImage() { - if (Registered) - deregisterWithDebugger(); - } - // Subclasses can override these methods to update the image with loaded - // addresses for sections and common symbols - void updateSectionAddress(const SectionRef &Sec, uint64_t Addr) override { - static_cast<DyldELFObject<ELFT>*>(getObjectFile()) - ->updateSectionAddress(Sec, Addr); - } +}; - void updateSymbolAddress(const SymbolRef &Sym, uint64_t Addr) override { - static_cast<DyldELFObject<ELFT>*>(getObjectFile()) - ->updateSymbolAddress(Sym, Addr); - } - void registerWithDebugger() override { - JITRegistrar::getGDBRegistrar().registerObject(*Buffer); - Registered = true; - } - void deregisterWithDebugger() override { - JITRegistrar::getGDBRegistrar().deregisterObject(*Buffer); - } -}; // The MemoryBuffer passed into this constructor is just a wrapper around the // actual memory. Ultimately, the Binary parent class will take ownership of // this MemoryBuffer object but not the underlying memory. template <class ELFT> -DyldELFObject<ELFT>::DyldELFObject(std::unique_ptr<MemoryBuffer> Wrapper, - std::error_code &EC) - : ELFObjectFile<ELFT>(std::move(Wrapper), EC) { - this->isDyldELFObject = true; -} - -template <class ELFT> -DyldELFObject<ELFT>::DyldELFObject(std::unique_ptr<ObjectFile> UnderlyingFile, - std::unique_ptr<MemoryBuffer> Wrapper, - std::error_code &EC) - : ELFObjectFile<ELFT>(std::move(Wrapper), EC), - UnderlyingFile(std::move(UnderlyingFile)) { +DyldELFObject<ELFT>::DyldELFObject(MemoryBufferRef Wrapper, std::error_code &EC) + : ELFObjectFile<ELFT>(Wrapper, EC) { this->isDyldELFObject = true; } @@ -148,10 +103,89 @@ void DyldELFObject<ELFT>::updateSymbolAddress(const SymbolRef &SymRef, sym->st_value = static_cast<addr_type>(Addr); } +class LoadedELFObjectInfo : public RuntimeDyld::LoadedObjectInfo { +public: + LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, + unsigned EndIdx) + : RuntimeDyld::LoadedObjectInfo(RTDyld, BeginIdx, EndIdx) {} + + OwningBinary<ObjectFile> + getObjectForDebug(const ObjectFile &Obj) const override; +}; + +template <typename ELFT> +std::unique_ptr<DyldELFObject<ELFT>> +createRTDyldELFObject(MemoryBufferRef Buffer, + const LoadedELFObjectInfo &L, + std::error_code &ec) { + typedef typename ELFFile<ELFT>::Elf_Shdr Elf_Shdr; + typedef typename ELFDataTypeTypedefHelper<ELFT>::value_type addr_type; + + std::unique_ptr<DyldELFObject<ELFT>> Obj = + llvm::make_unique<DyldELFObject<ELFT>>(Buffer, ec); + + // Iterate over all sections in the object. + for (const auto &Sec : Obj->sections()) { + StringRef SectionName; + Sec.getName(SectionName); + if (SectionName != "") { + DataRefImpl ShdrRef = Sec.getRawDataRefImpl(); + Elf_Shdr *shdr = const_cast<Elf_Shdr *>( + reinterpret_cast<const Elf_Shdr *>(ShdrRef.p)); + + if (uint64_t SecLoadAddr = L.getSectionLoadAddress(SectionName)) { + // This assumes that the address passed in matches the target address + // bitness. The template-based type cast handles everything else. + shdr->sh_addr = static_cast<addr_type>(SecLoadAddr); + } + } + } + + return Obj; +} + +OwningBinary<ObjectFile> createELFDebugObject(const ObjectFile &Obj, + const LoadedELFObjectInfo &L) { + assert(Obj.isELF() && "Not an ELF object file."); + + std::unique_ptr<MemoryBuffer> Buffer = + MemoryBuffer::getMemBufferCopy(Obj.getData(), Obj.getFileName()); + + std::error_code ec; + + std::unique_ptr<ObjectFile> DebugObj; + if (Obj.getBytesInAddress() == 4 && Obj.isLittleEndian()) { + typedef ELFType<support::little, 2, false> ELF32LE; + DebugObj = createRTDyldELFObject<ELF32LE>(Buffer->getMemBufferRef(), L, ec); + } else if (Obj.getBytesInAddress() == 4 && !Obj.isLittleEndian()) { + typedef ELFType<support::big, 2, false> ELF32BE; + DebugObj = createRTDyldELFObject<ELF32BE>(Buffer->getMemBufferRef(), L, ec); + } else if (Obj.getBytesInAddress() == 8 && !Obj.isLittleEndian()) { + typedef ELFType<support::big, 2, true> ELF64BE; + DebugObj = createRTDyldELFObject<ELF64BE>(Buffer->getMemBufferRef(), L, ec); + } else if (Obj.getBytesInAddress() == 8 && Obj.isLittleEndian()) { + typedef ELFType<support::little, 2, true> ELF64LE; + DebugObj = createRTDyldELFObject<ELF64LE>(Buffer->getMemBufferRef(), L, ec); + } else + llvm_unreachable("Unexpected ELF format"); + + assert(!ec && "Could not construct copy ELF object file"); + + return OwningBinary<ObjectFile>(std::move(DebugObj), std::move(Buffer)); +} + +OwningBinary<ObjectFile> +LoadedELFObjectInfo::getObjectForDebug(const ObjectFile &Obj) const { + return createELFDebugObject(Obj, *this); +} + } // namespace namespace llvm { +RuntimeDyldELF::RuntimeDyldELF(RTDyldMemoryManager *mm) : RuntimeDyldImpl(mm) {} +RuntimeDyldELF::~RuntimeDyldELF() {} + void RuntimeDyldELF::registerEHFrames() { if (!MemMgr) return; @@ -179,81 +213,14 @@ void RuntimeDyldELF::deregisterEHFrames() { RegisteredEHFrameSections.clear(); } -ObjectImage * -RuntimeDyldELF::createObjectImageFromFile(std::unique_ptr<object::ObjectFile> ObjFile) { - if (!ObjFile) - return nullptr; - - std::error_code ec; - std::unique_ptr<MemoryBuffer> Buffer( - MemoryBuffer::getMemBuffer(ObjFile->getData(), "", false)); - - if (ObjFile->getBytesInAddress() == 4 && ObjFile->isLittleEndian()) { - auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::little, 2, false>>>( - std::move(ObjFile), std::move(Buffer), ec); - return new ELFObjectImage<ELFType<support::little, 2, false>>( - nullptr, std::move(Obj)); - } else if (ObjFile->getBytesInAddress() == 4 && !ObjFile->isLittleEndian()) { - auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::big, 2, false>>>( - std::move(ObjFile), std::move(Buffer), ec); - return new ELFObjectImage<ELFType<support::big, 2, false>>(nullptr, std::move(Obj)); - } else if (ObjFile->getBytesInAddress() == 8 && !ObjFile->isLittleEndian()) { - auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::big, 2, true>>>( - std::move(ObjFile), std::move(Buffer), ec); - return new ELFObjectImage<ELFType<support::big, 2, true>>(nullptr, - std::move(Obj)); - } else if (ObjFile->getBytesInAddress() == 8 && ObjFile->isLittleEndian()) { - auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::little, 2, true>>>( - std::move(ObjFile), std::move(Buffer), ec); - return new ELFObjectImage<ELFType<support::little, 2, true>>( - nullptr, std::move(Obj)); - } else - llvm_unreachable("Unexpected ELF format"); -} - -ObjectImage *RuntimeDyldELF::createObjectImage(ObjectBuffer *Buffer) { - if (Buffer->getBufferSize() < ELF::EI_NIDENT) - llvm_unreachable("Unexpected ELF object size"); - std::pair<unsigned char, unsigned char> Ident = - std::make_pair((uint8_t)Buffer->getBufferStart()[ELF::EI_CLASS], - (uint8_t)Buffer->getBufferStart()[ELF::EI_DATA]); - std::error_code ec; - - std::unique_ptr<MemoryBuffer> Buf(Buffer->getMemBuffer()); - - if (Ident.first == ELF::ELFCLASS32 && Ident.second == ELF::ELFDATA2LSB) { - auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::little, 4, false>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::little, 4, false>>( - Buffer, std::move(Obj)); - } else if (Ident.first == ELF::ELFCLASS32 && - Ident.second == ELF::ELFDATA2MSB) { - auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::big, 4, false>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::big, 4, false>>(Buffer, - std::move(Obj)); - } else if (Ident.first == ELF::ELFCLASS64 && - Ident.second == ELF::ELFDATA2MSB) { - auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::big, 8, true>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::big, 8, true>>(Buffer, std::move(Obj)); - } else if (Ident.first == ELF::ELFCLASS64 && - Ident.second == ELF::ELFDATA2LSB) { - auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::little, 8, true>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::little, 8, true>>(Buffer, std::move(Obj)); - } else - llvm_unreachable("Unexpected ELF format"); +std::unique_ptr<RuntimeDyld::LoadedObjectInfo> +RuntimeDyldELF::loadObject(const object::ObjectFile &O) { + unsigned SectionStartIdx, SectionEndIdx; + std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O); + return llvm::make_unique<LoadedELFObjectInfo>(*this, SectionStartIdx, + SectionEndIdx); } -RuntimeDyldELF::~RuntimeDyldELF() {} - void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend, @@ -263,10 +230,9 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, llvm_unreachable("Relocation type not implemented yet!"); break; case ELF::R_X86_64_64: { - uint64_t *Target = reinterpret_cast<uint64_t *>(Section.Address + Offset); - *Target = Value + Addend; + support::ulittle64_t::ref(Section.Address + Offset) = Value + Addend; DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " - << format("%p\n", Target)); + << format("%p\n", Section.Address + Offset)); break; } case ELF::R_X86_64_32: @@ -276,17 +242,15 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, (Type == ELF::R_X86_64_32S && ((int64_t)Value <= INT32_MAX && (int64_t)Value >= INT32_MIN))); uint32_t TruncatedAddr = (Value & 0xFFFFFFFF); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); - *Target = TruncatedAddr; + support::ulittle32_t::ref(Section.Address + Offset) = TruncatedAddr; DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at " - << format("%p\n", Target)); + << format("%p\n", Section.Address + Offset)); break; } case ELF::R_X86_64_GOTPCREL: { // findGOTEntry returns the 'G + GOT' part of the relocation calculation // based on the load/target address of the GOT (not the current/local addr). uint64_t GOTAddr = findGOTEntry(Value, SymOffset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); uint64_t FinalAddress = Section.LoadAddress + Offset; // The processRelocationRef method combines the symbol offset and the addend // and in most cases that's what we want. For this relocation type, we need @@ -294,30 +258,29 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, int64_t RealOffset = GOTAddr + Addend - SymOffset - FinalAddress; assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN); int32_t TruncOffset = (RealOffset & 0xFFFFFFFF); - *Target = TruncOffset; + support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset; break; } case ELF::R_X86_64_PC32: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint32_t *Placeholder = - reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); + support::ulittle32_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); uint64_t FinalAddress = Section.LoadAddress + Offset; - int64_t RealOffset = *Placeholder + Value + Addend - FinalAddress; + int64_t RealOffset = Placeholder + Value + Addend - FinalAddress; assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN); int32_t TruncOffset = (RealOffset & 0xFFFFFFFF); - *Target = TruncOffset; + support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset; break; } case ELF::R_X86_64_PC64: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint64_t *Placeholder = - reinterpret_cast<uint64_t *>(Section.ObjAddress + Offset); - uint64_t *Target = reinterpret_cast<uint64_t *>(Section.Address + Offset); + support::ulittle64_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); uint64_t FinalAddress = Section.LoadAddress + Offset; - *Target = *Placeholder + Value + Addend - FinalAddress; + support::ulittle64_t::ref(Section.Address + Offset) = + Placeholder + Value + Addend - FinalAddress; break; } } @@ -330,21 +293,20 @@ void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section, case ELF::R_386_32: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint32_t *Placeholder = - reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); - *Target = *Placeholder + Value + Addend; + support::ulittle32_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); + support::ulittle32_t::ref(Section.Address + Offset) = + Placeholder + Value + Addend; break; } case ELF::R_386_PC32: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint32_t *Placeholder = - reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); + support::ulittle32_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); uint32_t FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF); - uint32_t RealOffset = *Placeholder + Value + Addend - FinalAddress; - *Target = RealOffset; + uint32_t RealOffset = Placeholder + Value + Addend - FinalAddress; + support::ulittle32_t::ref(Section.Address + Offset) = RealOffset; break; } default: @@ -617,7 +579,7 @@ void RuntimeDyldELF::resolveMIPSRelocation(const SectionEntry &Section, } // Return the .TOC. section and offset. -void RuntimeDyldELF::findPPC64TOCSection(ObjectImage &Obj, +void RuntimeDyldELF::findPPC64TOCSection(const ObjectFile &Obj, ObjSectionToIDMap &LocalSections, RelocationValueRef &Rel) { // Set a default SectionID in case we do not find a TOC section below. @@ -630,7 +592,7 @@ void RuntimeDyldELF::findPPC64TOCSection(ObjectImage &Obj, // The TOC consists of sections .got, .toc, .tocbss, .plt in that // order. The TOC starts where the first of these sections starts. - for (section_iterator si = Obj.begin_sections(), se = Obj.end_sections(); + for (section_iterator si = Obj.section_begin(), se = Obj.section_end(); si != se; ++si) { StringRef SectionName; @@ -652,15 +614,15 @@ void RuntimeDyldELF::findPPC64TOCSection(ObjectImage &Obj, // Returns the sections and offset associated with the ODP entry referenced // by Symbol. -void RuntimeDyldELF::findOPDEntrySection(ObjectImage &Obj, +void RuntimeDyldELF::findOPDEntrySection(const ObjectFile &Obj, ObjSectionToIDMap &LocalSections, RelocationValueRef &Rel) { // Get the ELF symbol value (st_value) to compare with Relocation offset in // .opd entries - for (section_iterator si = Obj.begin_sections(), se = Obj.end_sections(); + for (section_iterator si = Obj.section_begin(), se = Obj.section_end(); si != se; ++si) { section_iterator RelSecI = si->getRelocatedSection(); - if (RelSecI == Obj.end_sections()) + if (RelSecI == Obj.section_end()) continue; StringRef RelSectionName; @@ -702,10 +664,9 @@ void RuntimeDyldELF::findOPDEntrySection(ObjectImage &Obj, if (Rel.Addend != (int64_t)TargetSymbolOffset) continue; - section_iterator tsi(Obj.end_sections()); + section_iterator tsi(Obj.section_end()); check(TargetSymbol->getSection(tsi)); - bool IsCode = false; - tsi->isText(IsCode); + bool IsCode = tsi->isText(); Rel.SectionID = findOrEmitSection(Obj, (*tsi), IsCode, LocalSections); Rel.Addend = (intptr_t)Addend; return; @@ -911,8 +872,6 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section, break; case Triple::aarch64: case Triple::aarch64_be: - case Triple::arm64: - case Triple::arm64_be: resolveAArch64Relocation(Section, Offset, Value, Type, Addend); break; case Triple::arm: // Fall through. @@ -940,8 +899,9 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section, } relocation_iterator RuntimeDyldELF::processRelocationRef( - unsigned SectionID, relocation_iterator RelI, ObjectImage &Obj, - ObjSectionToIDMap &ObjSectionToID, const SymbolTableMap &Symbols, + unsigned SectionID, relocation_iterator RelI, + const ObjectFile &Obj, + ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) { uint64_t RelType; Check(RelI->getType(RelType)); @@ -951,75 +911,65 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( // Obtain the symbol name which is referenced in the relocation StringRef TargetName; - if (Symbol != Obj.end_symbols()) + if (Symbol != Obj.symbol_end()) Symbol->getName(TargetName); DEBUG(dbgs() << "\t\tRelType: " << RelType << " Addend: " << Addend << " TargetName: " << TargetName << "\n"); RelocationValueRef Value; // First search for the symbol in the local symbol table - SymbolTableMap::const_iterator lsi = Symbols.end(); SymbolRef::Type SymType = SymbolRef::ST_Unknown; - if (Symbol != Obj.end_symbols()) { - lsi = Symbols.find(TargetName.data()); + + // Search for the symbol in the global symbol table + SymbolTableMap::const_iterator gsi = GlobalSymbolTable.end(); + if (Symbol != Obj.symbol_end()) { + gsi = GlobalSymbolTable.find(TargetName.data()); Symbol->getType(SymType); } - if (lsi != Symbols.end()) { - Value.SectionID = lsi->second.first; - Value.Offset = lsi->second.second; - Value.Addend = lsi->second.second + Addend; + if (gsi != GlobalSymbolTable.end()) { + Value.SectionID = gsi->second.first; + Value.Offset = gsi->second.second; + Value.Addend = gsi->second.second + Addend; } else { - // Search for the symbol in the global symbol table - SymbolTableMap::const_iterator gsi = GlobalSymbolTable.end(); - if (Symbol != Obj.end_symbols()) - gsi = GlobalSymbolTable.find(TargetName.data()); - if (gsi != GlobalSymbolTable.end()) { - Value.SectionID = gsi->second.first; - Value.Offset = gsi->second.second; - Value.Addend = gsi->second.second + Addend; - } else { - switch (SymType) { - case SymbolRef::ST_Debug: { - // TODO: Now ELF SymbolRef::ST_Debug = STT_SECTION, it's not obviously - // and can be changed by another developers. Maybe best way is add - // a new symbol type ST_Section to SymbolRef and use it. - section_iterator si(Obj.end_sections()); - Symbol->getSection(si); - if (si == Obj.end_sections()) - llvm_unreachable("Symbol section not found, bad object file format!"); - DEBUG(dbgs() << "\t\tThis is section symbol\n"); - // Default to 'true' in case isText fails (though it never does). - bool isCode = true; - si->isText(isCode); - Value.SectionID = findOrEmitSection(Obj, (*si), isCode, ObjSectionToID); - Value.Addend = Addend; - break; - } - case SymbolRef::ST_Data: - case SymbolRef::ST_Unknown: { - Value.SymbolName = TargetName.data(); - Value.Addend = Addend; - - // Absolute relocations will have a zero symbol ID (STN_UNDEF), which - // will manifest here as a NULL symbol name. - // We can set this as a valid (but empty) symbol name, and rely - // on addRelocationForSymbol to handle this. - if (!Value.SymbolName) - Value.SymbolName = ""; - break; - } - default: - llvm_unreachable("Unresolved symbol type!"); - break; - } + switch (SymType) { + case SymbolRef::ST_Debug: { + // TODO: Now ELF SymbolRef::ST_Debug = STT_SECTION, it's not obviously + // and can be changed by another developers. Maybe best way is add + // a new symbol type ST_Section to SymbolRef and use it. + section_iterator si(Obj.section_end()); + Symbol->getSection(si); + if (si == Obj.section_end()) + llvm_unreachable("Symbol section not found, bad object file format!"); + DEBUG(dbgs() << "\t\tThis is section symbol\n"); + bool isCode = si->isText(); + Value.SectionID = findOrEmitSection(Obj, (*si), isCode, ObjSectionToID); + Value.Addend = Addend; + break; + } + case SymbolRef::ST_Data: + case SymbolRef::ST_Unknown: { + Value.SymbolName = TargetName.data(); + Value.Addend = Addend; + + // Absolute relocations will have a zero symbol ID (STN_UNDEF), which + // will manifest here as a NULL symbol name. + // We can set this as a valid (but empty) symbol name, and rely + // on addRelocationForSymbol to handle this. + if (!Value.SymbolName) + Value.SymbolName = ""; + break; + } + default: + llvm_unreachable("Unresolved symbol type!"); + break; } } + uint64_t Offset; Check(RelI->getOffset(Offset)); DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset << "\n"); - if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be || - Arch == Triple::arm64 || Arch == Triple::arm64_be) && + if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be) && (RelType == ELF::R_AARCH64_CALL26 || RelType == ELF::R_AARCH64_JUMP26)) { // This is an AArch64 branch relocation, need to use a stub function. DEBUG(dbgs() << "\t\tThis is an AArch64 branch relocation."); @@ -1143,7 +1093,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( if (RelType == ELF::R_PPC64_REL24) { // Determine ABI variant in use for this object. unsigned AbiVariant; - Obj.getObjectFile()->getPlatformFlags(AbiVariant); + Obj.getPlatformFlags(AbiVariant); AbiVariant &= ELF::EF_PPC64_ABI; // A PPC branch relocation will need a stub function if the target is // an external symbol (Symbol::ST_Unknown) or if the target address @@ -1323,7 +1273,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( Stubs[Value] = StubOffset; createStubFunction((uint8_t *)StubAddress); RelocationEntry RE(SectionID, StubOffset + 8, ELF::R_390_64, - Value.Addend - Addend); + Value.Offset); if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); else @@ -1431,8 +1381,6 @@ size_t RuntimeDyldELF::getGOTEntrySize() { case Triple::x86_64: case Triple::aarch64: case Triple::aarch64_be: - case Triple::arm64: - case Triple::arm64_be: case Triple::ppc64: case Triple::ppc64le: case Triple::systemz: @@ -1505,7 +1453,7 @@ uint64_t RuntimeDyldELF::findGOTEntry(uint64_t LoadAddress, uint64_t Offset) { return 0; } -void RuntimeDyldELF::finalizeLoad(ObjectImage &ObjImg, +void RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) { // If necessary, allocate the global offset table if (MemMgr) { @@ -1543,15 +1491,8 @@ void RuntimeDyldELF::finalizeLoad(ObjectImage &ObjImg, } } -bool RuntimeDyldELF::isCompatibleFormat(const ObjectBuffer *Buffer) const { - if (Buffer->getBufferSize() < strlen(ELF::ElfMagic)) - return false; - return (memcmp(Buffer->getBufferStart(), ELF::ElfMagic, - strlen(ELF::ElfMagic))) == 0; -} - -bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile *Obj) const { - return Obj->isELF(); +bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile &Obj) const { + return Obj.isELF(); } } // namespace llvm diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index 59fdfbe32521..b4414b08c480 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIME_DYLD_ELF_H -#define LLVM_RUNTIME_DYLD_ELF_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDELF_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDELF_H #include "RuntimeDyldImpl.h" #include "llvm/ADT/DenseMap.h" @@ -28,9 +28,11 @@ std::error_code Check(std::error_code Err) { } return Err; } + } // end anonymous namespace class RuntimeDyldELF : public RuntimeDyldImpl { + void resolveRelocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend, uint64_t SymOffset = 0); @@ -58,8 +60,7 @@ class RuntimeDyldELF : public RuntimeDyldImpl { uint64_t Value, uint32_t Type, int64_t Addend); unsigned getMaxStubSize() override { - if (Arch == Triple::aarch64 || Arch == Triple::arm64 || - Arch == Triple::aarch64_be || Arch == Triple::arm64_be) + if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) return 20; // movz; movk; movk; movk; br if (Arch == Triple::arm || Arch == Triple::thumb) return 8; // 32-bit instruction and 32-bit address @@ -82,9 +83,11 @@ class RuntimeDyldELF : public RuntimeDyldImpl { return 1; } - void findPPC64TOCSection(ObjectImage &Obj, ObjSectionToIDMap &LocalSections, + void findPPC64TOCSection(const ObjectFile &Obj, + ObjSectionToIDMap &LocalSections, RelocationValueRef &Rel); - void findOPDEntrySection(ObjectImage &Obj, ObjSectionToIDMap &LocalSections, + void findOPDEntrySection(const ObjectFile &Obj, + ObjSectionToIDMap &LocalSections, RelocationValueRef &Rel); uint64_t findGOTEntry(uint64_t LoadAddr, uint64_t Offset); @@ -105,23 +108,23 @@ class RuntimeDyldELF : public RuntimeDyldImpl { SmallVector<SID, 2> RegisteredEHFrameSections; public: - RuntimeDyldELF(RTDyldMemoryManager *mm) : RuntimeDyldImpl(mm) {} + RuntimeDyldELF(RTDyldMemoryManager *mm); + virtual ~RuntimeDyldELF(); + + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> + loadObject(const object::ObjectFile &O) override; void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override; relocation_iterator processRelocationRef(unsigned SectionID, relocation_iterator RelI, - ObjectImage &Obj, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols, StubMap &Stubs) override; - bool isCompatibleFormat(const ObjectBuffer *Buffer) const override; - bool isCompatibleFile(const object::ObjectFile *Buffer) const override; + const ObjectFile &Obj, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override; + bool isCompatibleFile(const object::ObjectFile &Obj) const override; void registerEHFrames() override; void deregisterEHFrames() override; - void finalizeLoad(ObjectImage &ObjImg, + void finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) override; - virtual ~RuntimeDyldELF(); - - static ObjectImage *createObjectImage(ObjectBuffer *InputBuffer); - static ObjectImage *createObjectImageFromFile(std::unique_ptr<object::ObjectFile> Obj); }; } // end namespace llvm diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 0211d2bbbb05..2f3e3a8f0344 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -11,14 +11,13 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIME_DYLD_IMPL_H -#define LLVM_RUNTIME_DYLD_IMPL_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDIMPL_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/Triple.h" -#include "llvm/ExecutionEngine/ObjectImage.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/ExecutionEngine/RuntimeDyldChecker.h" #include "llvm/Object/ObjectFile.h" @@ -37,7 +36,6 @@ using namespace llvm::object; namespace llvm { -class ObjectBuffer; class Twine; /// SectionEntry - represents a section emitted into memory by the dynamic @@ -70,7 +68,7 @@ public: SectionEntry(StringRef name, uint8_t *address, size_t size, uintptr_t objAddress) : Name(name), Address(address), Size(size), - LoadAddress((uintptr_t)address), StubOffset(size), + LoadAddress(reinterpret_cast<uintptr_t>(address)), StubOffset(size), ObjAddress(objAddress) {} }; @@ -159,19 +157,15 @@ public: }; class RuntimeDyldImpl { - friend class RuntimeDyldChecker; -private: - - uint64_t getAnySymbolRemoteAddress(StringRef Symbol) { - if (uint64_t InternalSymbolAddr = getSymbolLoadAddress(Symbol)) - return InternalSymbolAddr; - return MemMgr->getSymbolAddress(Symbol); - } - + friend class RuntimeDyld::LoadedObjectInfo; + friend class RuntimeDyldCheckerImpl; protected: // The MemoryManager to load objects into. RTDyldMemoryManager *MemMgr; + // Attached RuntimeDyldChecker instance. Null if no instance attached. + RuntimeDyldCheckerImpl *Checker; + // A list of all sections emitted by the dynamic linker. These sections are // referenced in the code by means of their index in this list - SectionID. typedef SmallVector<SectionEntry, 64> SectionList; @@ -211,6 +205,7 @@ protected: // modules. This map is indexed by symbol name. StringMap<RelocationList> ExternalSymbolRelocations; + typedef std::map<RelocationValueRef, uintptr_t> StubMap; Triple::ArchType Arch; @@ -245,11 +240,11 @@ protected: return true; } - uint64_t getSectionLoadAddress(unsigned SectionID) { + uint64_t getSectionLoadAddress(unsigned SectionID) const { return Sections[SectionID].LoadAddress; } - uint8_t *getSectionAddress(unsigned SectionID) { + uint8_t *getSectionAddress(unsigned SectionID) const { return (uint8_t *)Sections[SectionID].Address; } @@ -282,17 +277,25 @@ protected: *(Addr + 7) = Value & 0xFF; } + /// Endian-aware read Read the least significant Size bytes from Src. + uint64_t readBytesUnaligned(uint8_t *Src, unsigned Size) const; + + /// Endian-aware write. Write the least significant Size bytes from Value to + /// Dst. + void writeBytesUnaligned(uint64_t Value, uint8_t *Dst, unsigned Size) const; + /// \brief Given the common symbols discovered in the object file, emit a /// new section for them and update the symbol mappings in the object and /// symbol table. - void emitCommonSymbols(ObjectImage &Obj, const CommonSymbolMap &CommonSymbols, + void emitCommonSymbols(const ObjectFile &Obj, + const CommonSymbolMap &CommonSymbols, uint64_t TotalSize, SymbolTableMap &SymbolTable); /// \brief Emits section data from the object file to the MemoryManager. /// \param IsCode if it's true then allocateCodeSection() will be /// used for emits, else allocateDataSection() will be used. /// \return SectionID. - unsigned emitSection(ObjectImage &Obj, const SectionRef &Section, + unsigned emitSection(const ObjectFile &Obj, const SectionRef &Section, bool IsCode); /// \brief Find Section in LocalSections. If the secton is not found - emit @@ -300,7 +303,7 @@ protected: /// \param IsCode if it's true then allocateCodeSection() will be /// used for emmits, else allocateDataSection() will be used. /// \return SectionID. - unsigned findOrEmitSection(ObjectImage &Obj, const SectionRef &Section, + unsigned findOrEmitSection(const ObjectFile &Obj, const SectionRef &Section, bool IsCode, ObjSectionToIDMap &LocalSections); // \brief Add a relocation entry that uses the given section. @@ -328,8 +331,8 @@ protected: /// \return Iterator to the next relocation that needs to be parsed. virtual relocation_iterator processRelocationRef(unsigned SectionID, relocation_iterator RelI, - ObjectImage &Obj, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols, StubMap &Stubs) = 0; + const ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) = 0; /// \brief Resolve relocations to external symbols. void resolveExternalSymbols(); @@ -340,16 +343,19 @@ protected: // \brief Compute an upper bound of the memory that is required to load all // sections - void computeTotalAllocSize(ObjectImage &Obj, uint64_t &CodeSize, + void computeTotalAllocSize(const ObjectFile &Obj, uint64_t &CodeSize, uint64_t &DataSizeRO, uint64_t &DataSizeRW); // \brief Compute the stub buffer size required for a section - unsigned computeSectionStubBufSize(ObjectImage &Obj, + unsigned computeSectionStubBufSize(const ObjectFile &Obj, const SectionRef &Section); + // \brief Implementation of the generic part of the loadObject algorithm. + std::pair<unsigned, unsigned> loadObjectImpl(const object::ObjectFile &Obj); + public: RuntimeDyldImpl(RTDyldMemoryManager *mm) - : MemMgr(mm), ProcessAllSections(false), HasError(false) { + : MemMgr(mm), Checker(nullptr), ProcessAllSections(false), HasError(false) { } virtual ~RuntimeDyldImpl(); @@ -358,9 +364,14 @@ public: this->ProcessAllSections = ProcessAllSections; } - ObjectImage *loadObject(ObjectImage *InputObject); + void setRuntimeDyldChecker(RuntimeDyldCheckerImpl *Checker) { + this->Checker = Checker; + } + + virtual std::unique_ptr<RuntimeDyld::LoadedObjectInfo> + loadObject(const object::ObjectFile &Obj) = 0; - uint8_t* getSymbolAddress(StringRef Name) { + uint8_t* getSymbolAddress(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. SymbolTableMap::const_iterator pos = GlobalSymbolTable.find(Name); @@ -370,7 +381,7 @@ public: return getSectionAddress(Loc.first) + Loc.second; } - uint64_t getSymbolLoadAddress(StringRef Name) { + uint64_t getSymbolLoadAddress(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. SymbolTableMap::const_iterator pos = GlobalSymbolTable.find(Name); @@ -395,14 +406,14 @@ public: // Get the error message. StringRef getErrorString() { return ErrorStr; } - virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const = 0; - virtual bool isCompatibleFile(const ObjectFile *Obj) const = 0; + virtual bool isCompatibleFile(const ObjectFile &Obj) const = 0; virtual void registerEHFrames(); virtual void deregisterEHFrames(); - virtual void finalizeLoad(ObjectImage &ObjImg, ObjSectionToIDMap &SectionMap) {} + virtual void finalizeLoad(const ObjectFile &ObjImg, + ObjSectionToIDMap &SectionMap) {} }; } // end namespace llvm diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index 58fb51557c93..21893d2909ed 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -12,35 +12,49 @@ //===----------------------------------------------------------------------===// #include "RuntimeDyldMachO.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringRef.h" - -#include "Targets/RuntimeDyldMachOARM.h" #include "Targets/RuntimeDyldMachOAArch64.h" +#include "Targets/RuntimeDyldMachOARM.h" #include "Targets/RuntimeDyldMachOI386.h" #include "Targets/RuntimeDyldMachOX86_64.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" using namespace llvm; using namespace llvm::object; #define DEBUG_TYPE "dyld" +namespace { + +class LoadedMachOObjectInfo : public RuntimeDyld::LoadedObjectInfo { +public: + LoadedMachOObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, + unsigned EndIdx) + : RuntimeDyld::LoadedObjectInfo(RTDyld, BeginIdx, EndIdx) {} + + OwningBinary<ObjectFile> + getObjectForDebug(const ObjectFile &Obj) const override { + return OwningBinary<ObjectFile>(); + } +}; + +} + namespace llvm { -uint64_t RuntimeDyldMachO::decodeAddend(uint8_t *LocalAddress, unsigned NumBytes, - uint32_t RelType) const { - uint64_t Addend = 0; - memcpy(&Addend, LocalAddress, NumBytes); - return Addend; +int64_t RuntimeDyldMachO::memcpyAddend(const RelocationEntry &RE) const { + unsigned NumBytes = 1 << RE.Size; + uint8_t *Src = Sections[RE.SectionID].Address + RE.Offset; + + return static_cast<int64_t>(readBytesUnaligned(Src, NumBytes)); } RelocationValueRef RuntimeDyldMachO::getRelocationValueRef( - ObjectImage &ObjImg, const relocation_iterator &RI, - const RelocationEntry &RE, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols) { + const ObjectFile &BaseTObj, const relocation_iterator &RI, + const RelocationEntry &RE, ObjSectionToIDMap &ObjSectionToID) { const MachOObjectFile &Obj = - static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + static_cast<const MachOObjectFile &>(BaseTObj); MachO::any_relocation_info RelInfo = Obj.getRelocation(RI->getRawDataRefImpl()); RelocationValueRef Value; @@ -50,38 +64,32 @@ RelocationValueRef RuntimeDyldMachO::getRelocationValueRef( symbol_iterator Symbol = RI->getSymbol(); StringRef TargetName; Symbol->getName(TargetName); - SymbolTableMap::const_iterator SI = Symbols.find(TargetName.data()); - if (SI != Symbols.end()) { + SymbolTableMap::const_iterator SI = + GlobalSymbolTable.find(TargetName.data()); + if (SI != GlobalSymbolTable.end()) { Value.SectionID = SI->second.first; - Value.Addend = SI->second.second + RE.Addend; + Value.Offset = SI->second.second + RE.Addend; } else { - SI = GlobalSymbolTable.find(TargetName.data()); - if (SI != GlobalSymbolTable.end()) { - Value.SectionID = SI->second.first; - Value.Addend = SI->second.second + RE.Addend; - } else { - Value.SymbolName = TargetName.data(); - Value.Addend = RE.Addend; - } + Value.SymbolName = TargetName.data(); + Value.Offset = RE.Addend; } } else { SectionRef Sec = Obj.getRelocationSection(RelInfo); - bool IsCode = false; - Sec.isText(IsCode); - Value.SectionID = findOrEmitSection(ObjImg, Sec, IsCode, ObjSectionToID); - uint64_t Addr; - Sec.getAddress(Addr); - Value.Addend = RE.Addend - Addr; + bool IsCode = Sec.isText(); + Value.SectionID = findOrEmitSection(Obj, Sec, IsCode, ObjSectionToID); + uint64_t Addr = Sec.getAddress(); + Value.Offset = RE.Addend - Addr; } return Value; } void RuntimeDyldMachO::makeValueAddendPCRel(RelocationValueRef &Value, - ObjectImage &ObjImg, - const relocation_iterator &RI) { + const ObjectFile &BaseTObj, + const relocation_iterator &RI, + unsigned OffsetToNextPC) { const MachOObjectFile &Obj = - static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + static_cast<const MachOObjectFile &>(BaseTObj); MachO::any_relocation_info RelInfo = Obj.getRelocation(RI->getRawDataRefImpl()); @@ -89,8 +97,7 @@ void RuntimeDyldMachO::makeValueAddendPCRel(RelocationValueRef &Value, if (IsPCRel) { uint64_t RelocAddr = 0; RI->getAddress(RelocAddr); - unsigned RelocSize = Obj.getAnyRelocationLength(RelInfo); - Value.Addend += RelocAddr + (1ULL << RelocSize); + Value.Offset += RelocAddr + OffsetToNextPC; } } @@ -102,80 +109,142 @@ void RuntimeDyldMachO::dumpRelocationToResolve(const RelocationEntry &RE, dbgs() << "resolveRelocation Section: " << RE.SectionID << " LocalAddress: " << format("%p", LocalAddress) - << " FinalAddress: " << format("%p", FinalAddress) - << " Value: " << format("%p", Value) << " Addend: " << RE.Addend + << " FinalAddress: " << format("0x%016" PRIx64, FinalAddress) + << " Value: " << format("0x%016" PRIx64, Value) << " Addend: " << RE.Addend << " isPCRel: " << RE.IsPCRel << " MachoType: " << RE.RelType << " Size: " << (1 << RE.Size) << "\n"; } -bool RuntimeDyldMachO::writeBytesUnaligned(uint8_t *Addr, uint64_t Value, - unsigned Size) { - for (unsigned i = 0; i < Size; ++i) { - *Addr++ = (uint8_t)Value; - Value >>= 8; +section_iterator +RuntimeDyldMachO::getSectionByAddress(const MachOObjectFile &Obj, + uint64_t Addr) { + section_iterator SI = Obj.section_begin(); + section_iterator SE = Obj.section_end(); + + for (; SI != SE; ++SI) { + uint64_t SAddr = SI->getAddress(); + uint64_t SSize = SI->getSize(); + if ((Addr >= SAddr) && (Addr < SAddr + SSize)) + return SI; } - return false; + return SE; } -bool -RuntimeDyldMachO::isCompatibleFormat(const ObjectBuffer *InputBuffer) const { - if (InputBuffer->getBufferSize() < 4) - return false; - StringRef Magic(InputBuffer->getBufferStart(), 4); - if (Magic == "\xFE\xED\xFA\xCE") - return true; - if (Magic == "\xCE\xFA\xED\xFE") - return true; - if (Magic == "\xFE\xED\xFA\xCF") - return true; - if (Magic == "\xCF\xFA\xED\xFE") - return true; - return false; + +// Populate __pointers section. +void RuntimeDyldMachO::populateIndirectSymbolPointersSection( + const MachOObjectFile &Obj, + const SectionRef &PTSection, + unsigned PTSectionID) { + assert(!Obj.is64Bit() && + "Pointer table section not supported in 64-bit MachO."); + + MachO::dysymtab_command DySymTabCmd = Obj.getDysymtabLoadCommand(); + MachO::section Sec32 = Obj.getSection(PTSection.getRawDataRefImpl()); + uint32_t PTSectionSize = Sec32.size; + unsigned FirstIndirectSymbol = Sec32.reserved1; + const unsigned PTEntrySize = 4; + unsigned NumPTEntries = PTSectionSize / PTEntrySize; + unsigned PTEntryOffset = 0; + + assert((PTSectionSize % PTEntrySize) == 0 && + "Pointers section does not contain a whole number of stubs?"); + + DEBUG(dbgs() << "Populating pointer table section " + << Sections[PTSectionID].Name + << ", Section ID " << PTSectionID << ", " + << NumPTEntries << " entries, " << PTEntrySize + << " bytes each:\n"); + + for (unsigned i = 0; i < NumPTEntries; ++i) { + unsigned SymbolIndex = + Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i); + symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex); + StringRef IndirectSymbolName; + SI->getName(IndirectSymbolName); + DEBUG(dbgs() << " " << IndirectSymbolName << ": index " << SymbolIndex + << ", PT offset: " << PTEntryOffset << "\n"); + RelocationEntry RE(PTSectionID, PTEntryOffset, + MachO::GENERIC_RELOC_VANILLA, 0, false, 2); + addRelocationForSymbol(RE, IndirectSymbolName); + PTEntryOffset += PTEntrySize; + } +} + +bool RuntimeDyldMachO::isCompatibleFile(const object::ObjectFile &Obj) const { + return Obj.isMachO(); } -bool RuntimeDyldMachO::isCompatibleFile(const object::ObjectFile *Obj) const { - return Obj->isMachO(); +template <typename Impl> +void RuntimeDyldMachOCRTPBase<Impl>::finalizeLoad(const ObjectFile &ObjImg, + ObjSectionToIDMap &SectionMap) { + unsigned EHFrameSID = RTDYLD_INVALID_SECTION_ID; + unsigned TextSID = RTDYLD_INVALID_SECTION_ID; + unsigned ExceptTabSID = RTDYLD_INVALID_SECTION_ID; + ObjSectionToIDMap::iterator i, e; + + for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) { + const SectionRef &Section = i->first; + StringRef Name; + Section.getName(Name); + if (Name == "__eh_frame") + EHFrameSID = i->second; + else if (Name == "__text") + TextSID = i->second; + else if (Name == "__gcc_except_tab") + ExceptTabSID = i->second; + else + impl().finalizeSection(ObjImg, i->second, Section); + } + UnregisteredEHFrameSections.push_back( + EHFrameRelatedSections(EHFrameSID, TextSID, ExceptTabSID)); } -static unsigned char *processFDE(unsigned char *P, intptr_t DeltaForText, - intptr_t DeltaForEH) { +template <typename Impl> +unsigned char *RuntimeDyldMachOCRTPBase<Impl>::processFDE(unsigned char *P, + int64_t DeltaForText, + int64_t DeltaForEH) { + typedef typename Impl::TargetPtrT TargetPtrT; + DEBUG(dbgs() << "Processing FDE: Delta for text: " << DeltaForText << ", Delta for EH: " << DeltaForEH << "\n"); - uint32_t Length = *((uint32_t *)P); + uint32_t Length = readBytesUnaligned(P, 4); P += 4; unsigned char *Ret = P + Length; - uint32_t Offset = *((uint32_t *)P); + uint32_t Offset = readBytesUnaligned(P, 4); if (Offset == 0) // is a CIE return Ret; P += 4; - intptr_t FDELocation = *((intptr_t *)P); - intptr_t NewLocation = FDELocation - DeltaForText; - *((intptr_t *)P) = NewLocation; - P += sizeof(intptr_t); + TargetPtrT FDELocation = readBytesUnaligned(P, sizeof(TargetPtrT)); + TargetPtrT NewLocation = FDELocation - DeltaForText; + writeBytesUnaligned(NewLocation, P, sizeof(TargetPtrT)); + + P += sizeof(TargetPtrT); // Skip the FDE address range - P += sizeof(intptr_t); + P += sizeof(TargetPtrT); uint8_t Augmentationsize = *P; P += 1; if (Augmentationsize != 0) { - intptr_t LSDA = *((intptr_t *)P); - intptr_t NewLSDA = LSDA - DeltaForEH; - *((intptr_t *)P) = NewLSDA; + TargetPtrT LSDA = readBytesUnaligned(P, sizeof(TargetPtrT)); + TargetPtrT NewLSDA = LSDA - DeltaForEH; + writeBytesUnaligned(NewLSDA, P, sizeof(TargetPtrT)); } return Ret; } -static intptr_t computeDelta(SectionEntry *A, SectionEntry *B) { - intptr_t ObjDistance = A->ObjAddress - B->ObjAddress; - intptr_t MemDistance = A->LoadAddress - B->LoadAddress; +static int64_t computeDelta(SectionEntry *A, SectionEntry *B) { + int64_t ObjDistance = A->ObjAddress - B->ObjAddress; + int64_t MemDistance = A->LoadAddress - B->LoadAddress; return ObjDistance - MemDistance; } -void RuntimeDyldMachO::registerEHFrames() { +template <typename Impl> +void RuntimeDyldMachOCRTPBase<Impl>::registerEHFrames() { if (!MemMgr) return; @@ -190,8 +259,8 @@ void RuntimeDyldMachO::registerEHFrames() { if (SectionInfo.ExceptTabSID != RTDYLD_INVALID_SECTION_ID) ExceptTab = &Sections[SectionInfo.ExceptTabSID]; - intptr_t DeltaForText = computeDelta(Text, EHFrame); - intptr_t DeltaForEH = 0; + int64_t DeltaForText = computeDelta(Text, EHFrame); + int64_t DeltaForEH = 0; if (ExceptTab) DeltaForEH = computeDelta(ExceptTab, EHFrame); @@ -208,16 +277,24 @@ void RuntimeDyldMachO::registerEHFrames() { } std::unique_ptr<RuntimeDyldMachO> -llvm::RuntimeDyldMachO::create(Triple::ArchType Arch, RTDyldMemoryManager *MM) { +RuntimeDyldMachO::create(Triple::ArchType Arch, RTDyldMemoryManager *MM) { switch (Arch) { default: llvm_unreachable("Unsupported target for RuntimeDyldMachO."); break; case Triple::arm: return make_unique<RuntimeDyldMachOARM>(MM); - case Triple::arm64: return make_unique<RuntimeDyldMachOAArch64>(MM); + case Triple::aarch64: return make_unique<RuntimeDyldMachOAArch64>(MM); case Triple::x86: return make_unique<RuntimeDyldMachOI386>(MM); case Triple::x86_64: return make_unique<RuntimeDyldMachOX86_64>(MM); } } +std::unique_ptr<RuntimeDyld::LoadedObjectInfo> +RuntimeDyldMachO::loadObject(const object::ObjectFile &O) { + unsigned SectionStartIdx, SectionEndIdx; + std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O); + return llvm::make_unique<LoadedMachOObjectInfo>(*this, SectionStartIdx, + SectionEndIdx); +} + } // end namespace llvm diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h index 7d1dc0263db0..f8bfc03b6d22 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h @@ -11,10 +11,9 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIME_DYLD_MACHO_H -#define LLVM_RUNTIME_DYLD_MACHO_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDMACHO_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDMACHO_H -#include "ObjectImageCommon.h" #include "RuntimeDyldImpl.h" #include "llvm/Object/MachO.h" #include "llvm/Support/Format.h" @@ -37,6 +36,7 @@ protected: : EHFrameSID(RTDYLD_INVALID_SECTION_ID), TextSID(RTDYLD_INVALID_SECTION_ID), ExceptTabSID(RTDYLD_INVALID_SECTION_ID) {} + EHFrameRelatedSections(SID EH, SID T, SID Ex) : EHFrameSID(EH), TextSID(T), ExceptTabSID(Ex) {} SID EHFrameSID; @@ -51,9 +51,32 @@ protected: RuntimeDyldMachO(RTDyldMemoryManager *mm) : RuntimeDyldImpl(mm) {} - /// Extract the addend encoded in the instruction. - uint64_t decodeAddend(uint8_t *LocalAddress, unsigned NumBytes, - uint32_t RelType) const; + /// This convenience method uses memcpy to extract a contiguous addend (the + /// addend size and offset are taken from the corresponding fields of the RE). + int64_t memcpyAddend(const RelocationEntry &RE) const; + + /// Given a relocation_iterator for a non-scattered relocation, construct a + /// RelocationEntry and fill in the common fields. The 'Addend' field is *not* + /// filled in, since immediate encodings are highly target/opcode specific. + /// For targets/opcodes with simple, contiguous immediates (e.g. X86) the + /// memcpyAddend method can be used to read the immediate. + RelocationEntry getRelocationEntry(unsigned SectionID, + const ObjectFile &BaseTObj, + const relocation_iterator &RI) const { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(BaseTObj); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RI->getRawDataRefImpl()); + + bool IsPCRel = Obj.getAnyRelocationPCRel(RelInfo); + unsigned Size = Obj.getAnyRelocationLength(RelInfo); + uint64_t Offset; + RI->getOffset(Offset); + MachO::RelocationInfoType RelType = + static_cast<MachO::RelocationInfoType>(Obj.getAnyRelocationType(RelInfo)); + + return RelocationEntry(SectionID, Offset, RelType, 0, IsPCRel, Size); + } /// Construct a RelocationValueRef representing the relocation target. /// For Symbols in known sections, this will return a RelocationValueRef @@ -64,44 +87,42 @@ protected: /// In both cases the Addend field is *NOT* fixed up to be PC-relative. That /// should be done by the caller where appropriate by calling makePCRel on /// the RelocationValueRef. - RelocationValueRef getRelocationValueRef(ObjectImage &ObjImg, + RelocationValueRef getRelocationValueRef(const ObjectFile &BaseTObj, const relocation_iterator &RI, const RelocationEntry &RE, - ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols); + ObjSectionToIDMap &ObjSectionToID); /// Make the RelocationValueRef addend PC-relative. - void makeValueAddendPCRel(RelocationValueRef &Value, ObjectImage &ObjImg, - const relocation_iterator &RI); + void makeValueAddendPCRel(RelocationValueRef &Value, + const ObjectFile &BaseTObj, + const relocation_iterator &RI, + unsigned OffsetToNextPC); /// Dump information about the relocation entry (RE) and resolved value. void dumpRelocationToResolve(const RelocationEntry &RE, uint64_t Value) const; -public: - /// Create an ObjectImage from the given ObjectBuffer. - static ObjectImage *createObjectImage(ObjectBuffer *InputBuffer) { - return new ObjectImageCommon(InputBuffer); - } + // Return a section iterator for the section containing the given address. + static section_iterator getSectionByAddress(const MachOObjectFile &Obj, + uint64_t Addr); - /// Create an ObjectImage from the given ObjectFile. - static ObjectImage * - createObjectImageFromFile(std::unique_ptr<object::ObjectFile> InputObject) { - return new ObjectImageCommon(std::move(InputObject)); - } + + // Populate __pointers section. + void populateIndirectSymbolPointersSection(const MachOObjectFile &Obj, + const SectionRef &PTSection, + unsigned PTSectionID); + +public: /// Create a RuntimeDyldMachO instance for the given target architecture. static std::unique_ptr<RuntimeDyldMachO> create(Triple::ArchType Arch, RTDyldMemoryManager *mm); - /// Write the least significant 'Size' bytes in 'Value' out at the address - /// pointed to by Addr. Check for overflow. - bool writeBytesUnaligned(uint8_t *Addr, uint64_t Value, unsigned Size); + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> + loadObject(const object::ObjectFile &O) override; SectionEntry &getSection(unsigned SectionID) { return Sections[SectionID]; } - bool isCompatibleFormat(const ObjectBuffer *Buffer) const override; - bool isCompatibleFile(const object::ObjectFile *Obj) const override; - void registerEHFrames() override; + bool isCompatibleFile(const object::ObjectFile &Obj) const override; }; /// RuntimeDyldMachOTarget - Templated base class for generic MachO linker @@ -117,57 +138,15 @@ private: Impl &impl() { return static_cast<Impl &>(*this); } const Impl &impl() const { return static_cast<const Impl &>(*this); } -protected: - - /// Parse the given relocation, which must be a non-scattered, and - /// return a RelocationEntry representing the information. The 'Addend' field - /// will contain the unmodified instruction immediate. - RelocationEntry getBasicRelocationEntry(unsigned SectionID, - ObjectImage &ObjImg, - const relocation_iterator &RI) const { - const MachOObjectFile &Obj = - static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); - MachO::any_relocation_info RelInfo = - Obj.getRelocation(RI->getRawDataRefImpl()); - - const SectionEntry &Section = Sections[SectionID]; - bool IsPCRel = Obj.getAnyRelocationPCRel(RelInfo); - unsigned Size = Obj.getAnyRelocationLength(RelInfo); - uint64_t Offset; - RI->getOffset(Offset); - uint8_t *LocalAddress = Section.Address + Offset; - unsigned NumBytes = 1 << Size; - uint32_t RelType = Obj.getAnyRelocationType(RelInfo); - uint64_t Addend = impl().decodeAddend(LocalAddress, NumBytes, RelType); - - return RelocationEntry(SectionID, Offset, RelType, Addend, IsPCRel, Size); - } + unsigned char *processFDE(unsigned char *P, int64_t DeltaForText, + int64_t DeltaForEH); public: RuntimeDyldMachOCRTPBase(RTDyldMemoryManager *mm) : RuntimeDyldMachO(mm) {} - void finalizeLoad(ObjectImage &ObjImg, ObjSectionToIDMap &SectionMap) { - unsigned EHFrameSID = RTDYLD_INVALID_SECTION_ID; - unsigned TextSID = RTDYLD_INVALID_SECTION_ID; - unsigned ExceptTabSID = RTDYLD_INVALID_SECTION_ID; - ObjSectionToIDMap::iterator i, e; - - for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) { - const SectionRef &Section = i->first; - StringRef Name; - Section.getName(Name); - if (Name == "__eh_frame") - EHFrameSID = i->second; - else if (Name == "__text") - TextSID = i->second; - else if (Name == "__gcc_except_tab") - ExceptTabSID = i->second; - else - impl().finalizeSection(ObjImg, i->second, Section); - } - UnregisteredEHFrameSections.push_back( - EHFrameRelatedSections(EHFrameSID, TextSID, ExceptTabSID)); - } + void finalizeLoad(const ObjectFile &Obj, + ObjSectionToIDMap &SectionMap) override; + void registerEHFrames() override; }; } // end namespace llvm diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h index 775ed9ec3635..196fa62a0a07 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -7,10 +7,11 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIMEDYLDMACHOAARCH64_H -#define LLVM_RUNTIMEDYLDMACHOAARCH64_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOAARCH64_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOAARCH64_H #include "../RuntimeDyldMachO.h" +#include "llvm/Support/Endian.h" #define DEBUG_TYPE "dyld" @@ -19,6 +20,9 @@ namespace llvm { class RuntimeDyldMachOAArch64 : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOAArch64> { public: + + typedef uint64_t TargetPtrT; + RuntimeDyldMachOAArch64(RTDyldMemoryManager *MM) : RuntimeDyldMachOCRTPBase(MM) {} @@ -26,12 +30,224 @@ public: unsigned getStubAlignment() override { return 8; } + /// Extract the addend encoded in the instruction / memory location. + int64_t decodeAddend(const RelocationEntry &RE) const { + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + unsigned NumBytes = 1 << RE.Size; + int64_t Addend = 0; + // Verify that the relocation has the correct size and alignment. + switch (RE.RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + assert((NumBytes == 4 || NumBytes == 8) && "Invalid relocation size."); + break; + case MachO::ARM64_RELOC_BRANCH26: + case MachO::ARM64_RELOC_PAGE21: + case MachO::ARM64_RELOC_PAGEOFF12: + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: + assert(NumBytes == 4 && "Invalid relocation size."); + assert((((uintptr_t)LocalAddress & 0x3) == 0) && + "Instruction address is not aligned to 4 bytes."); + break; + } + + switch (RE.RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + // This could be an unaligned memory location. + if (NumBytes == 4) + Addend = *reinterpret_cast<support::ulittle32_t *>(LocalAddress); + else + Addend = *reinterpret_cast<support::ulittle64_t *>(LocalAddress); + break; + case MachO::ARM64_RELOC_BRANCH26: { + // Verify that the relocation points to the expected branch instruction. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0xFC000000) == 0x14000000 && "Expected branch instruction."); + + // Get the 26 bit addend encoded in the branch instruction and sign-extend + // to 64 bit. The lower 2 bits are always zeros and are therefore implicit + // (<< 2). + Addend = (*p & 0x03FFFFFF) << 2; + Addend = SignExtend64(Addend, 28); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_PAGE21: { + // Verify that the relocation points to the expected adrp instruction. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0x9F000000) == 0x90000000 && "Expected adrp instruction."); + + // Get the 21 bit addend encoded in the adrp instruction and sign-extend + // to 64 bit. The lower 12 bits (4096 byte page) are always zeros and are + // therefore implicit (<< 12). + Addend = ((*p & 0x60000000) >> 29) | ((*p & 0x01FFFFE0) >> 3) << 12; + Addend = SignExtend64(Addend, 33); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + (void)p; + assert((*p & 0x3B000000) == 0x39000000 && + "Only expected load / store instructions."); + } // fall-through + case MachO::ARM64_RELOC_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // or add / sub instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((((*p & 0x3B000000) == 0x39000000) || + ((*p & 0x11C00000) == 0x11000000) ) && + "Expected load / store or add/sub instruction."); + + // Get the 12 bit addend encoded in the instruction. + Addend = (*p & 0x003FFC00) >> 10; + + // Check which instruction we are decoding to obtain the implicit shift + // factor of the instruction. + int ImplicitShift = 0; + if ((*p & 0x3B000000) == 0x39000000) { // << load / store + // For load / store instructions the size is encoded in bits 31:30. + ImplicitShift = ((*p >> 30) & 0x3); + if (ImplicitShift == 0) { + // Check if this a vector op to get the correct shift value. + if ((*p & 0x04800000) == 0x04800000) + ImplicitShift = 4; + } + } + // Compensate for implicit shift. + Addend <<= ImplicitShift; + break; + } + } + return Addend; + } + + /// Extract the addend encoded in the instruction. + void encodeAddend(uint8_t *LocalAddress, unsigned NumBytes, + MachO::RelocationInfoType RelType, int64_t Addend) const { + // Verify that the relocation has the correct alignment. + switch (RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + assert((NumBytes == 4 || NumBytes == 8) && "Invalid relocation size."); + break; + case MachO::ARM64_RELOC_BRANCH26: + case MachO::ARM64_RELOC_PAGE21: + case MachO::ARM64_RELOC_PAGEOFF12: + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: + assert(NumBytes == 4 && "Invalid relocation size."); + assert((((uintptr_t)LocalAddress & 0x3) == 0) && + "Instruction address is not aligned to 4 bytes."); + break; + } + + switch (RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + // This could be an unaligned memory location. + if (NumBytes == 4) + *reinterpret_cast<support::ulittle32_t *>(LocalAddress) = Addend; + else + *reinterpret_cast<support::ulittle64_t *>(LocalAddress) = Addend; + break; + case MachO::ARM64_RELOC_BRANCH26: { + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + // Verify that the relocation points to the expected branch instruction. + assert((*p & 0xFC000000) == 0x14000000 && "Expected branch instruction."); + + // Verify addend value. + assert((Addend & 0x3) == 0 && "Branch target is not aligned"); + assert(isInt<28>(Addend) && "Branch target is out of range."); + + // Encode the addend as 26 bit immediate in the branch instruction. + *p = (*p & 0xFC000000) | ((uint32_t)(Addend >> 2) & 0x03FFFFFF); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_PAGE21: { + // Verify that the relocation points to the expected adrp instruction. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0x9F000000) == 0x90000000 && "Expected adrp instruction."); + + // Check that the addend fits into 21 bits (+ 12 lower bits). + assert((Addend & 0xFFF) == 0 && "ADRP target is not page aligned."); + assert(isInt<33>(Addend) && "Invalid page reloc value."); + + // Encode the addend into the instruction. + uint32_t ImmLoValue = ((uint64_t)Addend << 17) & 0x60000000; + uint32_t ImmHiValue = ((uint64_t)Addend >> 9) & 0x00FFFFE0; + *p = (*p & 0x9F00001F) | ImmHiValue | ImmLoValue; + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0x3B000000) == 0x39000000 && + "Only expected load / store instructions."); + (void)p; + } // fall-through + case MachO::ARM64_RELOC_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // or add / sub instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((((*p & 0x3B000000) == 0x39000000) || + ((*p & 0x11C00000) == 0x11000000) ) && + "Expected load / store or add/sub instruction."); + + // Check which instruction we are decoding to obtain the implicit shift + // factor of the instruction and verify alignment. + int ImplicitShift = 0; + if ((*p & 0x3B000000) == 0x39000000) { // << load / store + // For load / store instructions the size is encoded in bits 31:30. + ImplicitShift = ((*p >> 30) & 0x3); + switch (ImplicitShift) { + case 0: + // Check if this a vector op to get the correct shift value. + if ((*p & 0x04800000) == 0x04800000) { + ImplicitShift = 4; + assert(((Addend & 0xF) == 0) && + "128-bit LDR/STR not 16-byte aligned."); + } + break; + case 1: + assert(((Addend & 0x1) == 0) && "16-bit LDR/STR not 2-byte aligned."); + break; + case 2: + assert(((Addend & 0x3) == 0) && "32-bit LDR/STR not 4-byte aligned."); + break; + case 3: + assert(((Addend & 0x7) == 0) && "64-bit LDR/STR not 8-byte aligned."); + break; + } + } + // Compensate for implicit shift. + Addend >>= ImplicitShift; + assert(isUInt<12>(Addend) && "Addend cannot be encoded."); + + // Encode the addend into the instruction. + *p = (*p & 0xFFC003FF) | ((uint32_t)(Addend << 10) & 0x003FFC00); + break; + } + } + } + relocation_iterator processRelocationRef(unsigned SectionID, relocation_iterator RelI, - ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols, StubMap &Stubs) override { + const ObjectFile &BaseObjT, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { const MachOObjectFile &Obj = - static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + static_cast<const MachOObjectFile &>(BaseObjT); MachO::any_relocation_info RelInfo = Obj.getRelocation(RelI->getRawDataRefImpl()); @@ -41,34 +257,35 @@ public: // addend for the following relocation. If found: (1) store the associated // addend, (2) consume the next relocation, and (3) use the stored addend to // override the addend. - bool HasExplicitAddend = false; int64_t ExplicitAddend = 0; if (Obj.getAnyRelocationType(RelInfo) == MachO::ARM64_RELOC_ADDEND) { assert(!Obj.getPlainRelocationExternal(RelInfo)); assert(!Obj.getAnyRelocationPCRel(RelInfo)); assert(Obj.getAnyRelocationLength(RelInfo) == 2); - HasExplicitAddend = true; int64_t RawAddend = Obj.getPlainRelocationSymbolNum(RelInfo); // Sign-extend the 24-bit to 64-bit. - ExplicitAddend = (RawAddend << 40) >> 40; + ExplicitAddend = SignExtend64(RawAddend, 24); ++RelI; RelInfo = Obj.getRelocation(RelI->getRawDataRefImpl()); } - RelocationEntry RE(getBasicRelocationEntry(SectionID, ObjImg, RelI)); + RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI)); + RE.Addend = decodeAddend(RE); RelocationValueRef Value( - getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + getRelocationValueRef(Obj, RelI, RE, ObjSectionToID)); - if (HasExplicitAddend) { + assert((ExplicitAddend == 0 || RE.Addend == 0) && "Relocation has "\ + "ARM64_RELOC_ADDEND and embedded addend in the instruction."); + if (ExplicitAddend) { RE.Addend = ExplicitAddend; - Value.Addend = ExplicitAddend; + Value.Offset = ExplicitAddend; } bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); if (!IsExtern && RE.IsPCRel) - makeValueAddendPCRel(Value, ObjImg, RelI); + makeValueAddendPCRel(Value, Obj, RelI, 1 << RE.Size); - RE.Addend = Value.Addend; + RE.Addend = Value.Offset; if (RE.RelType == MachO::ARM64_RELOC_GOT_LOAD_PAGE21 || RE.RelType == MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12) @@ -83,13 +300,15 @@ public: return ++RelI; } - void resolveRelocation(const RelocationEntry &RE, uint64_t Value) { + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.Address + RE.Offset; + MachO::RelocationInfoType RelType = + static_cast<MachO::RelocationInfoType>(RE.RelType); - switch (RE.RelType) { + switch (RelType) { default: llvm_unreachable("Invalid relocation type!"); case MachO::ARM64_RELOC_UNSIGNED: { @@ -99,117 +318,49 @@ public: if (RE.Size < 2) llvm_unreachable("Invalid size for ARM64_RELOC_UNSIGNED"); - writeBytesUnaligned(LocalAddress, Value + RE.Addend, 1 << RE.Size); + encodeAddend(LocalAddress, 1 << RE.Size, RelType, Value + RE.Addend); break; } case MachO::ARM64_RELOC_BRANCH26: { assert(RE.IsPCRel && "not PCRel and ARM64_RELOC_BRANCH26 not supported"); - // Mask the value into the target address. We know instructions are - // 32-bit aligned, so we can do it all at once. - uint32_t *p = (uint32_t *)LocalAddress; - // Check if the addend is encoded in the instruction. - uint32_t EncodedAddend = *p & 0x03FFFFFF; - if (EncodedAddend != 0) { - if (RE.Addend == 0) - llvm_unreachable("branch26 instruction has embedded addend."); - else - llvm_unreachable("branch26 instruction has embedded addend and" - "ARM64_RELOC_ADDEND."); - } // Check if branch is in range. uint64_t FinalAddress = Section.LoadAddress + RE.Offset; - uint64_t PCRelVal = Value - FinalAddress + RE.Addend; - assert(isInt<26>(PCRelVal) && "Branch target out of range!"); - // Insert the value into the instruction. - *p = (*p & 0xFC000000) | ((uint32_t)(PCRelVal >> 2) & 0x03FFFFFF); + int64_t PCRelVal = Value - FinalAddress + RE.Addend; + encodeAddend(LocalAddress, /*Size=*/4, RelType, PCRelVal); break; } case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: case MachO::ARM64_RELOC_PAGE21: { assert(RE.IsPCRel && "not PCRel and ARM64_RELOC_PAGE21 not supported"); - // Mask the value into the target address. We know instructions are - // 32-bit aligned, so we can do it all at once. - uint32_t *p = (uint32_t *)LocalAddress; - // Check if the addend is encoded in the instruction. - uint32_t EncodedAddend = - ((*p & 0x60000000) >> 29) | ((*p & 0x01FFFFE0) >> 3); - if (EncodedAddend != 0) { - if (RE.Addend == 0) - llvm_unreachable("adrp instruction has embedded addend."); - else - llvm_unreachable("adrp instruction has embedded addend and" - "ARM64_RELOC_ADDEND."); - } // Adjust for PC-relative relocation and offset. uint64_t FinalAddress = Section.LoadAddress + RE.Offset; - uint64_t PCRelVal = - ((Value + RE.Addend) & (-4096)) - (FinalAddress & (-4096)); - // Check that the value fits into 21 bits (+ 12 lower bits). - assert(isInt<33>(PCRelVal) && "Invalid page reloc value!"); - // Insert the value into the instruction. - uint32_t ImmLoValue = (uint32_t)(PCRelVal << 17) & 0x60000000; - uint32_t ImmHiValue = (uint32_t)(PCRelVal >> 9) & 0x00FFFFE0; - *p = (*p & 0x9F00001F) | ImmHiValue | ImmLoValue; + int64_t PCRelVal = + ((Value + RE.Addend) & (-4096)) - (FinalAddress & (-4096)); + encodeAddend(LocalAddress, /*Size=*/4, RelType, PCRelVal); break; } case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: case MachO::ARM64_RELOC_PAGEOFF12: { assert(!RE.IsPCRel && "PCRel and ARM64_RELOC_PAGEOFF21 not supported"); - // Mask the value into the target address. We know instructions are - // 32-bit aligned, so we can do it all at once. - uint32_t *p = (uint32_t *)LocalAddress; - // Check if the addend is encoded in the instruction. - uint32_t EncodedAddend = *p & 0x003FFC00; - if (EncodedAddend != 0) { - if (RE.Addend == 0) - llvm_unreachable("adrp instruction has embedded addend."); - else - llvm_unreachable("adrp instruction has embedded addend and" - "ARM64_RELOC_ADDEND."); - } // Add the offset from the symbol. Value += RE.Addend; // Mask out the page address and only use the lower 12 bits. Value &= 0xFFF; - // Check which instruction we are updating to obtain the implicit shift - // factor from LDR/STR instructions. - if (*p & 0x08000000) { - uint32_t ImplicitShift = ((*p >> 30) & 0x3); - switch (ImplicitShift) { - case 0: - // Check if this a vector op. - if ((*p & 0x04800000) == 0x04800000) { - ImplicitShift = 4; - assert(((Value & 0xF) == 0) && - "128-bit LDR/STR not 16-byte aligned."); - } - break; - case 1: - assert(((Value & 0x1) == 0) && "16-bit LDR/STR not 2-byte aligned."); - case 2: - assert(((Value & 0x3) == 0) && "32-bit LDR/STR not 4-byte aligned."); - case 3: - assert(((Value & 0x7) == 0) && "64-bit LDR/STR not 8-byte aligned."); - } - // Compensate for implicit shift. - Value >>= ImplicitShift; - } - // Insert the value into the instruction. - *p = (*p & 0xFFC003FF) | ((uint32_t)(Value << 10) & 0x003FFC00); + encodeAddend(LocalAddress, /*Size=*/4, RelType, Value); break; } case MachO::ARM64_RELOC_SUBTRACTOR: case MachO::ARM64_RELOC_POINTER_TO_GOT: case MachO::ARM64_RELOC_TLVP_LOAD_PAGE21: case MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12: - llvm_unreachable("Relocation type not implemented yet!"); + llvm_unreachable("Relocation type not yet implemented!"); case MachO::ARM64_RELOC_ADDEND: llvm_unreachable("ARM64_RELOC_ADDEND should have been handeled by " "processRelocationRef!"); } } - void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, + void finalizeSection(const ObjectFile &Obj, unsigned SectionID, const SectionRef &Section) {} private: @@ -218,9 +369,9 @@ private: assert(RE.Size == 2); SectionEntry &Section = Sections[RE.SectionID]; StubMap::const_iterator i = Stubs.find(Value); - uint8_t *Addr; + int64_t Offset; if (i != Stubs.end()) - Addr = Section.Address + i->second; + Offset = static_cast<int64_t>(i->second); else { // FIXME: There must be a better way to do this then to check and fix the // alignment every time!!! @@ -234,22 +385,22 @@ private: assert(((StubAddress % getStubAlignment()) == 0) && "GOT entry not aligned"); RelocationEntry GOTRE(RE.SectionID, StubOffset, - MachO::ARM64_RELOC_UNSIGNED, Value.Addend, + MachO::ARM64_RELOC_UNSIGNED, Value.Offset, /*IsPCRel=*/false, /*Size=*/3); if (Value.SymbolName) addRelocationForSymbol(GOTRE, Value.SymbolName); else addRelocationForSection(GOTRE, Value.SectionID); Section.StubOffset = StubOffset + getMaxStubSize(); - Addr = (uint8_t *)StubAddress; + Offset = static_cast<int64_t>(StubOffset); } - RelocationEntry TargetRE(RE.SectionID, RE.Offset, RE.RelType, /*Addend=*/0, + RelocationEntry TargetRE(RE.SectionID, RE.Offset, RE.RelType, Offset, RE.IsPCRel, RE.Size); - resolveRelocation(TargetRE, (uint64_t)Addr); + addRelocationForSection(TargetRE, RE.SectionID); } }; } #undef DEBUG_TYPE -#endif // LLVM_RUNTIMEDYLDMACHOAARCH64_H +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index 1de994219824..09e430e22619 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIMEDYLDMACHOARM_H -#define LLVM_RUNTIMEDYLDMACHOARM_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOARM_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOARM_H #include "../RuntimeDyldMachO.h" @@ -18,37 +18,66 @@ namespace llvm { class RuntimeDyldMachOARM : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOARM> { +private: + typedef RuntimeDyldMachOCRTPBase<RuntimeDyldMachOARM> ParentT; + public: + + typedef uint32_t TargetPtrT; + RuntimeDyldMachOARM(RTDyldMemoryManager *MM) : RuntimeDyldMachOCRTPBase(MM) {} unsigned getMaxStubSize() override { return 8; } unsigned getStubAlignment() override { return 4; } + int64_t decodeAddend(const RelocationEntry &RE) const { + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + + switch (RE.RelType) { + default: + return memcpyAddend(RE); + case MachO::ARM_RELOC_BR24: { + uint32_t Temp = readBytesUnaligned(LocalAddress, 4); + Temp &= 0x00ffffff; // Mask out the opcode. + // Now we've got the shifted immediate, shift by 2, sign extend and ret. + return SignExtend32<26>(Temp << 2); + } + } + } + relocation_iterator processRelocationRef(unsigned SectionID, relocation_iterator RelI, - ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols, StubMap &Stubs) override { + const ObjectFile &BaseObjT, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { const MachOObjectFile &Obj = - static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + static_cast<const MachOObjectFile &>(BaseObjT); MachO::any_relocation_info RelInfo = Obj.getRelocation(RelI->getRawDataRefImpl()); + uint32_t RelType = Obj.getAnyRelocationType(RelInfo); - if (Obj.isRelocationScattered(RelInfo)) - return ++++RelI; + if (Obj.isRelocationScattered(RelInfo)) { + if (RelType == MachO::ARM_RELOC_HALF_SECTDIFF) + return processHALFSECTDIFFRelocation(SectionID, RelI, Obj, + ObjSectionToID); + else + return ++++RelI; + } - RelocationEntry RE(getBasicRelocationEntry(SectionID, ObjImg, RelI)); + RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI)); + RE.Addend = decodeAddend(RE); RelocationValueRef Value( - getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + getRelocationValueRef(Obj, RelI, RE, ObjSectionToID)); - bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); - if (!IsExtern && RE.IsPCRel) - makeValueAddendPCRel(Value, ObjImg, RelI); + if (RE.IsPCRel) + makeValueAddendPCRel(Value, Obj, RelI, 8); if ((RE.RelType & 0xf) == MachO::ARM_RELOC_BR24) processBranchRelocation(RE, Value, Stubs); else { - RE.Addend = Value.Addend; + RE.Addend = Value.Offset; if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); else @@ -58,7 +87,7 @@ public: return ++RelI; } - void resolveRelocation(const RelocationEntry &RE, uint64_t Value) { + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.Address + RE.Offset; @@ -78,33 +107,45 @@ public: default: llvm_unreachable("Invalid relocation type!"); case MachO::ARM_RELOC_VANILLA: - writeBytesUnaligned(LocalAddress, Value, 1 << RE.Size); + writeBytesUnaligned(Value + RE.Addend, LocalAddress, 1 << RE.Size); break; case MachO::ARM_RELOC_BR24: { // Mask the value into the target address. We know instructions are // 32-bit aligned, so we can do it all at once. - uint32_t *p = (uint32_t *)LocalAddress; + Value += RE.Addend; // The low two bits of the value are not encoded. Value >>= 2; // Mask the value to 24 bits. uint64_t FinalValue = Value & 0xffffff; - // Check for overflow. - if (Value != FinalValue) { - Error("ARM BR24 relocation out of range."); - return; - } // FIXME: If the destination is a Thumb function (and the instruction // is a non-predicated BL instruction), we need to change it to a BLX // instruction instead. // Insert the value into the instruction. - *p = (*p & ~0xffffff) | FinalValue; + uint32_t Temp = readBytesUnaligned(LocalAddress, 4); + writeBytesUnaligned((Temp & ~0xffffff) | FinalValue, LocalAddress, 4); + + break; + } + case MachO::ARM_RELOC_HALF_SECTDIFF: { + uint64_t SectionABase = Sections[RE.Sections.SectionA].LoadAddress; + uint64_t SectionBBase = Sections[RE.Sections.SectionB].LoadAddress; + assert((Value == SectionABase || Value == SectionBBase) && + "Unexpected HALFSECTDIFF relocation value."); + Value = SectionABase - SectionBBase + RE.Addend; + if (RE.Size & 0x1) // :upper16: + Value = (Value >> 16); + Value &= 0xffff; + + uint32_t Insn = readBytesUnaligned(LocalAddress, 4); + Insn = (Insn & 0xfff0f000) | ((Value & 0xf000) << 4) | (Value & 0x0fff); + writeBytesUnaligned(Insn, LocalAddress, 4); break; } + case MachO::ARM_THUMB_RELOC_BR22: case MachO::ARM_THUMB_32BIT_BRANCH: case MachO::ARM_RELOC_HALF: - case MachO::ARM_RELOC_HALF_SECTDIFF: case MachO::ARM_RELOC_PAIR: case MachO::ARM_RELOC_SECTDIFF: case MachO::ARM_RELOC_LOCAL_SECTDIFF: @@ -114,10 +155,18 @@ public: } } - void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, - const SectionRef &Section) {} + void finalizeSection(const ObjectFile &Obj, unsigned SectionID, + const SectionRef &Section) { + StringRef Name; + Section.getName(Name); + + if (Name == "__nl_symbol_ptr") + populateIndirectSymbolPointersSection(cast<MachOObjectFile>(Obj), + Section, SectionID); + } private: + void processBranchRelocation(const RelocationEntry &RE, const RelocationValueRef &Value, StubMap &Stubs) { @@ -134,7 +183,8 @@ private: uint8_t *StubTargetAddr = createStubFunction(Section.Address + Section.StubOffset); RelocationEntry StubRE(RE.SectionID, StubTargetAddr - Section.Address, - MachO::GENERIC_RELOC_VANILLA, Value.Addend); + MachO::GENERIC_RELOC_VANILLA, Value.Offset, false, + 2); if (Value.SymbolName) addRelocationForSymbol(StubRE, Value.SymbolName); else @@ -142,13 +192,86 @@ private: Addr = Section.Address + Section.StubOffset; Section.StubOffset += getMaxStubSize(); } - RelocationEntry TargetRE(Value.SectionID, RE.Offset, RE.RelType, 0, + RelocationEntry TargetRE(RE.SectionID, RE.Offset, RE.RelType, 0, RE.IsPCRel, RE.Size); resolveRelocation(TargetRE, (uint64_t)Addr); } + + relocation_iterator + processHALFSECTDIFFRelocation(unsigned SectionID, relocation_iterator RelI, + const ObjectFile &BaseTObj, + ObjSectionToIDMap &ObjSectionToID) { + const MachOObjectFile &MachO = + static_cast<const MachOObjectFile&>(BaseTObj); + MachO::any_relocation_info RE = + MachO.getRelocation(RelI->getRawDataRefImpl()); + + + // For a half-diff relocation the length bits actually record whether this + // is a movw/movt, and whether this is arm or thumb. + // Bit 0 indicates movw (b0 == 0) or movt (b0 == 1). + // Bit 1 indicates arm (b1 == 0) or thumb (b1 == 1). + unsigned HalfDiffKindBits = MachO.getAnyRelocationLength(RE); + if (HalfDiffKindBits & 0x2) + llvm_unreachable("Thumb not yet supported."); + + SectionEntry &Section = Sections[SectionID]; + uint32_t RelocType = MachO.getAnyRelocationType(RE); + bool IsPCRel = MachO.getAnyRelocationPCRel(RE); + uint64_t Offset; + RelI->getOffset(Offset); + uint8_t *LocalAddress = Section.Address + Offset; + int64_t Immediate = readBytesUnaligned(LocalAddress, 4); // Copy the whole instruction out. + Immediate = ((Immediate >> 4) & 0xf000) | (Immediate & 0xfff); + + ++RelI; + MachO::any_relocation_info RE2 = + MachO.getRelocation(RelI->getRawDataRefImpl()); + uint32_t AddrA = MachO.getScatteredRelocationValue(RE); + section_iterator SAI = getSectionByAddress(MachO, AddrA); + assert(SAI != MachO.section_end() && "Can't find section for address A"); + uint64_t SectionABase = SAI->getAddress(); + uint64_t SectionAOffset = AddrA - SectionABase; + SectionRef SectionA = *SAI; + bool IsCode = SectionA.isText(); + uint32_t SectionAID = + findOrEmitSection(MachO, SectionA, IsCode, ObjSectionToID); + + uint32_t AddrB = MachO.getScatteredRelocationValue(RE2); + section_iterator SBI = getSectionByAddress(MachO, AddrB); + assert(SBI != MachO.section_end() && "Can't find section for address B"); + uint64_t SectionBBase = SBI->getAddress(); + uint64_t SectionBOffset = AddrB - SectionBBase; + SectionRef SectionB = *SBI; + uint32_t SectionBID = + findOrEmitSection(MachO, SectionB, IsCode, ObjSectionToID); + + uint32_t OtherHalf = MachO.getAnyRelocationAddress(RE2) & 0xffff; + unsigned Shift = (HalfDiffKindBits & 0x1) ? 16 : 0; + uint32_t FullImmVal = (Immediate << Shift) | (OtherHalf << (16 - Shift)); + int64_t Addend = FullImmVal - (AddrA - AddrB); + + // addend = Encoded - Expected + // = Encoded - (AddrA - AddrB) + + DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA << ", AddrB: " << AddrB + << ", Addend: " << Addend << ", SectionA ID: " << SectionAID + << ", SectionAOffset: " << SectionAOffset + << ", SectionB ID: " << SectionBID + << ", SectionBOffset: " << SectionBOffset << "\n"); + RelocationEntry R(SectionID, Offset, RelocType, Addend, SectionAID, + SectionAOffset, SectionBID, SectionBOffset, IsPCRel, + HalfDiffKindBits); + + addRelocationForSection(R, SectionAID); + addRelocationForSection(R, SectionBID); + + return ++RelI; + } + }; } #undef DEBUG_TYPE -#endif // LLVM_RUNTIMEDYLDMACHOARM_H +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index 856c6ca3035c..67d7027c1858 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIMEDYLDMACHOI386_H -#define LLVM_RUNTIMEDYLDMACHOI386_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOI386_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOI386_H #include "../RuntimeDyldMachO.h" @@ -19,6 +19,9 @@ namespace llvm { class RuntimeDyldMachOI386 : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOI386> { public: + + typedef uint32_t TargetPtrT; + RuntimeDyldMachOI386(RTDyldMemoryManager *MM) : RuntimeDyldMachOCRTPBase(MM) {} @@ -28,10 +31,11 @@ public: relocation_iterator processRelocationRef(unsigned SectionID, relocation_iterator RelI, - ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols, StubMap &Stubs) override { + const ObjectFile &BaseObjT, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { const MachOObjectFile &Obj = - static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + static_cast<const MachOObjectFile &>(BaseObjT); MachO::any_relocation_info RelInfo = Obj.getRelocation(RelI->getRawDataRefImpl()); uint32_t RelType = Obj.getAnyRelocationType(RelInfo); @@ -39,17 +43,18 @@ public: if (Obj.isRelocationScattered(RelInfo)) { if (RelType == MachO::GENERIC_RELOC_SECTDIFF || RelType == MachO::GENERIC_RELOC_LOCAL_SECTDIFF) - return processSECTDIFFRelocation(SectionID, RelI, ObjImg, + return processSECTDIFFRelocation(SectionID, RelI, Obj, ObjSectionToID); - else if (Arch == Triple::x86 && RelType == MachO::GENERIC_RELOC_VANILLA) - return processI386ScatteredVANILLA(SectionID, RelI, ObjImg, + else if (RelType == MachO::GENERIC_RELOC_VANILLA) + return processI386ScatteredVANILLA(SectionID, RelI, Obj, ObjSectionToID); llvm_unreachable("Unhandled scattered relocation."); } - RelocationEntry RE(getBasicRelocationEntry(SectionID, ObjImg, RelI)); + RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI)); + RE.Addend = memcpyAddend(RE); RelocationValueRef Value( - getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + getRelocationValueRef(Obj, RelI, RE, ObjSectionToID)); // Addends for external, PC-rel relocations on i386 point back to the zero // offset. Calculate the final offset from the relocation target instead. @@ -62,9 +67,9 @@ public: // Value.Addend += RelocAddr + 4; // } if (RE.IsPCRel) - makeValueAddendPCRel(Value, ObjImg, RelI); + makeValueAddendPCRel(Value, Obj, RelI, 1 << RE.Size); - RE.Addend = Value.Addend; + RE.Addend = Value.Offset; if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); @@ -74,7 +79,7 @@ public: return ++RelI; } - void resolveRelocation(const RelocationEntry &RE, uint64_t Value) { + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; @@ -89,7 +94,7 @@ public: default: llvm_unreachable("Invalid relocation type!"); case MachO::GENERIC_RELOC_VANILLA: - writeBytesUnaligned(LocalAddress, Value + RE.Addend, 1 << RE.Size); + writeBytesUnaligned(Value + RE.Addend, LocalAddress, 1 << RE.Size); break; case MachO::GENERIC_RELOC_SECTDIFF: case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: { @@ -98,7 +103,7 @@ public: assert((Value == SectionABase || Value == SectionBBase) && "Unexpected SECTDIFF relocation value."); Value = SectionABase - SectionBBase + RE.Addend; - writeBytesUnaligned(LocalAddress, Value, 1 << RE.Size); + writeBytesUnaligned(Value, LocalAddress, 1 << RE.Size); break; } case MachO::GENERIC_RELOC_PB_LA_PTR: @@ -106,61 +111,56 @@ public: } } - void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, + void finalizeSection(const ObjectFile &Obj, unsigned SectionID, const SectionRef &Section) { StringRef Name; Section.getName(Name); if (Name == "__jump_table") - populateJumpTable(cast<MachOObjectFile>(*ObjImg.getObjectFile()), Section, - SectionID); + populateJumpTable(cast<MachOObjectFile>(Obj), Section, SectionID); else if (Name == "__pointers") - populatePointersSection(cast<MachOObjectFile>(*ObjImg.getObjectFile()), - Section, SectionID); + populateIndirectSymbolPointersSection(cast<MachOObjectFile>(Obj), + Section, SectionID); } private: relocation_iterator processSECTDIFFRelocation(unsigned SectionID, relocation_iterator RelI, - ObjectImage &Obj, + const ObjectFile &BaseObjT, ObjSectionToIDMap &ObjSectionToID) { - const MachOObjectFile *MachO = - static_cast<const MachOObjectFile *>(Obj.getObjectFile()); + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile&>(BaseObjT); MachO::any_relocation_info RE = - MachO->getRelocation(RelI->getRawDataRefImpl()); + Obj.getRelocation(RelI->getRawDataRefImpl()); SectionEntry &Section = Sections[SectionID]; - uint32_t RelocType = MachO->getAnyRelocationType(RE); - bool IsPCRel = MachO->getAnyRelocationPCRel(RE); - unsigned Size = MachO->getAnyRelocationLength(RE); + uint32_t RelocType = Obj.getAnyRelocationType(RE); + bool IsPCRel = Obj.getAnyRelocationPCRel(RE); + unsigned Size = Obj.getAnyRelocationLength(RE); uint64_t Offset; RelI->getOffset(Offset); uint8_t *LocalAddress = Section.Address + Offset; unsigned NumBytes = 1 << Size; - int64_t Addend = 0; - memcpy(&Addend, LocalAddress, NumBytes); + uint64_t Addend = readBytesUnaligned(LocalAddress, NumBytes); ++RelI; MachO::any_relocation_info RE2 = - MachO->getRelocation(RelI->getRawDataRefImpl()); + Obj.getRelocation(RelI->getRawDataRefImpl()); - uint32_t AddrA = MachO->getScatteredRelocationValue(RE); - section_iterator SAI = getSectionByAddress(*MachO, AddrA); - assert(SAI != MachO->section_end() && "Can't find section for address A"); - uint64_t SectionABase; - SAI->getAddress(SectionABase); + uint32_t AddrA = Obj.getScatteredRelocationValue(RE); + section_iterator SAI = getSectionByAddress(Obj, AddrA); + assert(SAI != Obj.section_end() && "Can't find section for address A"); + uint64_t SectionABase = SAI->getAddress(); uint64_t SectionAOffset = AddrA - SectionABase; SectionRef SectionA = *SAI; - bool IsCode; - SectionA.isText(IsCode); + bool IsCode = SectionA.isText(); uint32_t SectionAID = findOrEmitSection(Obj, SectionA, IsCode, ObjSectionToID); - uint32_t AddrB = MachO->getScatteredRelocationValue(RE2); - section_iterator SBI = getSectionByAddress(*MachO, AddrB); - assert(SBI != MachO->section_end() && "Can't find section for address B"); - uint64_t SectionBBase; - SBI->getAddress(SectionBBase); + uint32_t AddrB = Obj.getScatteredRelocationValue(RE2); + section_iterator SBI = getSectionByAddress(Obj, AddrB); + assert(SBI != Obj.section_end() && "Can't find section for address B"); + uint64_t SectionBBase = SBI->getAddress(); uint64_t SectionBOffset = AddrB - SectionBBase; SectionRef SectionB = *SBI; uint32_t SectionBID = @@ -185,32 +185,30 @@ private: } relocation_iterator processI386ScatteredVANILLA( - unsigned SectionID, relocation_iterator RelI, ObjectImage &Obj, + unsigned SectionID, relocation_iterator RelI, + const ObjectFile &BaseObjT, RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID) { - const MachOObjectFile *MachO = - static_cast<const MachOObjectFile *>(Obj.getObjectFile()); + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile&>(BaseObjT); MachO::any_relocation_info RE = - MachO->getRelocation(RelI->getRawDataRefImpl()); + Obj.getRelocation(RelI->getRawDataRefImpl()); SectionEntry &Section = Sections[SectionID]; - uint32_t RelocType = MachO->getAnyRelocationType(RE); - bool IsPCRel = MachO->getAnyRelocationPCRel(RE); - unsigned Size = MachO->getAnyRelocationLength(RE); + uint32_t RelocType = Obj.getAnyRelocationType(RE); + bool IsPCRel = Obj.getAnyRelocationPCRel(RE); + unsigned Size = Obj.getAnyRelocationLength(RE); uint64_t Offset; RelI->getOffset(Offset); uint8_t *LocalAddress = Section.Address + Offset; unsigned NumBytes = 1 << Size; - int64_t Addend = 0; - memcpy(&Addend, LocalAddress, NumBytes); - - unsigned SymbolBaseAddr = MachO->getScatteredRelocationValue(RE); - section_iterator TargetSI = getSectionByAddress(*MachO, SymbolBaseAddr); - assert(TargetSI != MachO->section_end() && "Can't find section for symbol"); - uint64_t SectionBaseAddr; - TargetSI->getAddress(SectionBaseAddr); + int64_t Addend = readBytesUnaligned(LocalAddress, NumBytes); + + unsigned SymbolBaseAddr = Obj.getScatteredRelocationValue(RE); + section_iterator TargetSI = getSectionByAddress(Obj, SymbolBaseAddr); + assert(TargetSI != Obj.section_end() && "Can't find section for symbol"); + uint64_t SectionBaseAddr = TargetSI->getAddress(); SectionRef TargetSection = *TargetSI; - bool IsCode; - TargetSection.isText(IsCode); + bool IsCode = TargetSection.isText(); uint32_t TargetSectionID = findOrEmitSection(Obj, TargetSection, IsCode, ObjSectionToID); @@ -223,7 +221,7 @@ private: } // Populate stubs in __jump_table section. - void populateJumpTable(MachOObjectFile &Obj, const SectionRef &JTSection, + void populateJumpTable(const MachOObjectFile &Obj, const SectionRef &JTSection, unsigned JTSectionID) { assert(!Obj.is64Bit() && "__jump_table section not supported in 64-bit MachO."); @@ -255,61 +253,9 @@ private: } } - // Populate __pointers section. - void populatePointersSection(MachOObjectFile &Obj, - const SectionRef &PTSection, - unsigned PTSectionID) { - assert(!Obj.is64Bit() && - "__pointers section not supported in 64-bit MachO."); - - MachO::dysymtab_command DySymTabCmd = Obj.getDysymtabLoadCommand(); - MachO::section Sec32 = Obj.getSection(PTSection.getRawDataRefImpl()); - uint32_t PTSectionSize = Sec32.size; - unsigned FirstIndirectSymbol = Sec32.reserved1; - const unsigned PTEntrySize = 4; - unsigned NumPTEntries = PTSectionSize / PTEntrySize; - unsigned PTEntryOffset = 0; - - assert((PTSectionSize % PTEntrySize) == 0 && - "Pointers section does not contain a whole number of stubs?"); - - DEBUG(dbgs() << "Populating __pointers, Section ID " << PTSectionID << ", " - << NumPTEntries << " entries, " << PTEntrySize - << " bytes each:\n"); - - for (unsigned i = 0; i < NumPTEntries; ++i) { - unsigned SymbolIndex = - Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i); - symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex); - StringRef IndirectSymbolName; - SI->getName(IndirectSymbolName); - DEBUG(dbgs() << " " << IndirectSymbolName << ": index " << SymbolIndex - << ", PT offset: " << PTEntryOffset << "\n"); - RelocationEntry RE(PTSectionID, PTEntryOffset, - MachO::GENERIC_RELOC_VANILLA, 0, false, 2); - addRelocationForSymbol(RE, IndirectSymbolName); - PTEntryOffset += PTEntrySize; - } - } - - static section_iterator getSectionByAddress(const MachOObjectFile &Obj, - uint64_t Addr) { - section_iterator SI = Obj.section_begin(); - section_iterator SE = Obj.section_end(); - - for (; SI != SE; ++SI) { - uint64_t SAddr, SSize; - SI->getAddress(SAddr); - SI->getSize(SSize); - if ((Addr >= SAddr) && (Addr < SAddr + SSize)) - return SI; - } - - return SE; - } }; } #undef DEBUG_TYPE -#endif // LLVM_RUNTIMEDYLDMACHOI386_H +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h index 99efe9da4486..0734017e2204 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIMEDYLDMACHOX86_64_H -#define LLVM_RUNTIMEDYLDMACHOX86_64_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOX86_64_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOX86_64_H #include "../RuntimeDyldMachO.h" @@ -19,6 +19,9 @@ namespace llvm { class RuntimeDyldMachOX86_64 : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOX86_64> { public: + + typedef uint64_t TargetPtrT; + RuntimeDyldMachOX86_64(RTDyldMemoryManager *MM) : RuntimeDyldMachOCRTPBase(MM) {} @@ -28,29 +31,31 @@ public: relocation_iterator processRelocationRef(unsigned SectionID, relocation_iterator RelI, - ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols, StubMap &Stubs) override { + const ObjectFile &BaseObjT, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { const MachOObjectFile &Obj = - static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + static_cast<const MachOObjectFile &>(BaseObjT); MachO::any_relocation_info RelInfo = Obj.getRelocation(RelI->getRawDataRefImpl()); assert(!Obj.isRelocationScattered(RelInfo) && "Scattered relocations not supported on X86_64"); - RelocationEntry RE(getBasicRelocationEntry(SectionID, ObjImg, RelI)); + RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI)); + RE.Addend = memcpyAddend(RE); RelocationValueRef Value( - getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + getRelocationValueRef(Obj, RelI, RE, ObjSectionToID)); bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); if (!IsExtern && RE.IsPCRel) - makeValueAddendPCRel(Value, ObjImg, RelI); + makeValueAddendPCRel(Value, Obj, RelI, 1 << RE.Size); if (RE.RelType == MachO::X86_64_RELOC_GOT || RE.RelType == MachO::X86_64_RELOC_GOT_LOAD) processGOTRelocation(RE, Value, Stubs); else { - RE.Addend = Value.Addend; + RE.Addend = Value.Offset; if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); else @@ -60,7 +65,7 @@ public: return ++RelI; } - void resolveRelocation(const RelocationEntry &RE, uint64_t Value) { + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.Address + RE.Offset; @@ -83,7 +88,7 @@ public: case MachO::X86_64_RELOC_SIGNED: case MachO::X86_64_RELOC_UNSIGNED: case MachO::X86_64_RELOC_BRANCH: - writeBytesUnaligned(LocalAddress, Value + RE.Addend, 1 << RE.Size); + writeBytesUnaligned(Value + RE.Addend, LocalAddress, 1 << RE.Size); break; case MachO::X86_64_RELOC_GOT_LOAD: case MachO::X86_64_RELOC_GOT: @@ -93,7 +98,7 @@ public: } } - void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, + void finalizeSection(const ObjectFile &Obj, unsigned SectionID, const SectionRef &Section) {} private: @@ -102,7 +107,7 @@ private: SectionEntry &Section = Sections[RE.SectionID]; assert(RE.IsPCRel); assert(RE.Size == 2); - Value.Addend -= RE.Addend; + Value.Offset -= RE.Addend; RuntimeDyldMachO::StubMap::const_iterator i = Stubs.find(Value); uint8_t *Addr; if (i != Stubs.end()) { @@ -111,7 +116,7 @@ private: Stubs[Value] = Section.StubOffset; uint8_t *GOTEntry = Section.Address + Section.StubOffset; RelocationEntry GOTRE(RE.SectionID, Section.StubOffset, - MachO::X86_64_RELOC_UNSIGNED, Value.Addend, false, + MachO::X86_64_RELOC_UNSIGNED, Value.Offset, false, 3); if (Value.SymbolName) addRelocationForSymbol(GOTRE, Value.SymbolName); @@ -129,4 +134,4 @@ private: #undef DEBUG_TYPE -#endif // LLVM_RUNTIMEDYLDMACHOX86_64_H +#endif diff --git a/lib/ExecutionEngine/TargetSelect.cpp b/lib/ExecutionEngine/TargetSelect.cpp index b10d51f64862..e6679cfb7f74 100644 --- a/lib/ExecutionEngine/TargetSelect.cpp +++ b/lib/ExecutionEngine/TargetSelect.cpp @@ -30,7 +30,7 @@ TargetMachine *EngineBuilder::selectTarget() { // MCJIT can generate code for remote targets, but the old JIT and Interpreter // must use the host architecture. - if (UseMCJIT && WhichEngine != EngineKind::Interpreter && M) + if (WhichEngine != EngineKind::Interpreter && M) TT.setTriple(M->getTargetTriple()); return selectTarget(TT, MArch, MCPU, MAttrs); @@ -89,8 +89,7 @@ TargetMachine *EngineBuilder::selectTarget(const Triple &TargetTriple, } // FIXME: non-iOS ARM FastISel is broken with MCJIT. - if (UseMCJIT && - TheTriple.getArch() == Triple::arm && + if (TheTriple.getArch() == Triple::arm && !TheTriple.isiOS() && OptLevel == CodeGenOpt::None) { OptLevel = CodeGenOpt::Less; |