diff options
Diffstat (limited to 'contrib/llvm-project/compiler-rt/lib/orc')
49 files changed, 8879 insertions, 0 deletions
diff --git a/contrib/llvm-project/compiler-rt/lib/orc/adt.h b/contrib/llvm-project/compiler-rt/lib/orc/adt.h new file mode 100644 index 000000000000..8884cc8812be --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/adt.h @@ -0,0 +1,63 @@ +//===----------------------- adt.h - Handy ADTs -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_ADT_H +#define ORC_RT_ADT_H + +#include <cstring> +#include <limits> +#include <ostream> +#include <string> + +namespace __orc_rt { + +constexpr std::size_t dynamic_extent = std::numeric_limits<std::size_t>::max(); + +/// A substitute for std::span (and llvm::ArrayRef). +/// FIXME: Remove in favor of std::span once we can use c++20. +template <typename T, std::size_t Extent = dynamic_extent> class span { +public: + typedef T element_type; + typedef std::remove_cv<T> value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef T *pointer; + typedef const T *const_pointer; + typedef T &reference; + typedef const T &const_reference; + + typedef pointer iterator; + + static constexpr std::size_t extent = Extent; + + constexpr span() noexcept = default; + constexpr span(T *first, size_type count) noexcept + : Data(first), Size(count) {} + + template <std::size_t N> + constexpr span(T (&arr)[N]) noexcept : Data(&arr[0]), Size(N) {} + + constexpr iterator begin() const noexcept { return Data; } + constexpr iterator end() const noexcept { return Data + Size; } + constexpr pointer data() const noexcept { return Data; } + constexpr reference operator[](size_type idx) const { return Data[idx]; } + constexpr size_type size() const noexcept { return Size; } + constexpr bool empty() const noexcept { return Size == 0; } + +private: + T *Data = nullptr; + size_type Size = 0; +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_ADT_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/bitmask_enum.h b/contrib/llvm-project/compiler-rt/lib/orc/bitmask_enum.h new file mode 100644 index 000000000000..b9fb776bdf23 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/bitmask_enum.h @@ -0,0 +1,151 @@ +//===---- bitmask_enum.h - Enable bitmask operations on enums ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_BITMASK_ENUM_H +#define ORC_RT_BITMASK_ENUM_H + +#include "stl_extras.h" + +#include <cassert> +#include <type_traits> + +namespace __orc_rt { + +/// ORC_RT_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you +/// can perform bitwise operations on it without putting static_cast everywhere. +/// +/// \code +/// enum MyEnum { +/// E1 = 1, E2 = 2, E3 = 4, E4 = 8, +/// ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ E4) +/// }; +/// +/// void Foo() { +/// MyEnum A = (E1 | E2) & E3 ^ ~E4; // Look, ma: No static_cast! +/// } +/// \endcode +/// +/// Normally when you do a bitwise operation on an enum value, you get back an +/// instance of the underlying type (e.g. int). But using this macro, bitwise +/// ops on your enum will return you back instances of the enum. This is +/// particularly useful for enums which represent a combination of flags. +/// +/// The parameter to ORC_RT_MARK_AS_BITMASK_ENUM should be the largest +/// individual value in your enum. +/// +/// All of the enum's values must be non-negative. +#define ORC_RT_MARK_AS_BITMASK_ENUM(LargestValue) \ + ORC_RT_BITMASK_LARGEST_ENUMERATOR = LargestValue + +/// ORC_RT_DECLARE_ENUM_AS_BITMASK can be used to declare an enum type as a bit +/// set, so that bitwise operation on such enum does not require static_cast. +/// +/// \code +/// enum MyEnum { E1 = 1, E2 = 2, E3 = 4, E4 = 8 }; +/// ORC_RT_DECLARE_ENUM_AS_BITMASK(MyEnum, E4); +/// +/// void Foo() { +/// MyEnum A = (E1 | E2) & E3 ^ ~E4; // No static_cast +/// } +/// \endcode +/// +/// The second parameter to ORC_RT_DECLARE_ENUM_AS_BITMASK specifies the largest +/// bit value of the enum type. +/// +/// ORC_RT_DECLARE_ENUM_AS_BITMASK should be used in __orc_rt namespace. +/// +/// This a non-intrusive alternative for ORC_RT_MARK_AS_BITMASK_ENUM. It allows +/// declaring more than one non-scoped enumerations as bitmask types in the same +/// scope. Otherwise it provides the same functionality as +/// ORC_RT_MARK_AS_BITMASK_ENUM. +#define ORC_RT_DECLARE_ENUM_AS_BITMASK(Enum, LargestValue) \ + template <> struct is_bitmask_enum<Enum> : std::true_type {}; \ + template <> struct largest_bitmask_enum_bit<Enum> { \ + static constexpr std::underlying_type_t<Enum> value = LargestValue; \ + } + +/// Traits class to determine whether an enum has been declared as a bitwise +/// enum via ORC_RT_DECLARE_ENUM_AS_BITMASK. +template <typename E, typename Enable = void> +struct is_bitmask_enum : std::false_type {}; + +template <typename E> +struct is_bitmask_enum< + E, std::enable_if_t<sizeof(E::ORC_RT_BITMASK_LARGEST_ENUMERATOR) >= 0>> + : std::true_type {}; + +template <typename E> +inline constexpr bool is_bitmask_enum_v = is_bitmask_enum<E>::value; + +/// Traits class to deermine bitmask enum largest bit. +template <typename E, typename Enable = void> struct largest_bitmask_enum_bit; + +template <typename E> +struct largest_bitmask_enum_bit< + E, std::enable_if_t<sizeof(E::ORC_RT_BITMASK_LARGEST_ENUMERATOR) >= 0>> { + using UnderlyingTy = std::underlying_type_t<E>; + static constexpr UnderlyingTy value = + static_cast<UnderlyingTy>(E::ORC_RT_BITMASK_LARGEST_ENUMERATOR); +}; + +template <typename E> constexpr std::underlying_type_t<E> Mask() { + return bit_ceil(largest_bitmask_enum_bit<E>::value) - 1; +} + +template <typename E> constexpr std::underlying_type_t<E> Underlying(E Val) { + auto U = static_cast<std::underlying_type_t<E>>(Val); + assert(U >= 0 && "Negative enum values are not allowed"); + assert(U <= Mask<E>() && "Enum value too large (or langest val too small"); + return U; +} + +template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>> +constexpr E operator~(E Val) { + return static_cast<E>(~Underlying(Val) & Mask<E>()); +} + +template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>> +constexpr E operator|(E LHS, E RHS) { + return static_cast<E>(Underlying(LHS) | Underlying(RHS)); +} + +template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>> +constexpr E operator&(E LHS, E RHS) { + return static_cast<E>(Underlying(LHS) & Underlying(RHS)); +} + +template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>> +constexpr E operator^(E LHS, E RHS) { + return static_cast<E>(Underlying(LHS) ^ Underlying(RHS)); +} + +template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>> +E &operator|=(E &LHS, E RHS) { + LHS = LHS | RHS; + return LHS; +} + +template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>> +E &operator&=(E &LHS, E RHS) { + LHS = LHS & RHS; + return LHS; +} + +template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>> +E &operator^=(E &LHS, E RHS) { + LHS = LHS ^ RHS; + return LHS; +} + +} // end namespace __orc_rt + +#endif // ORC_RT_BITMASK_ENUM_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.cpp b/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.cpp new file mode 100644 index 000000000000..9fe5c0b06289 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.cpp @@ -0,0 +1,775 @@ +//===- coff_platform.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code required to load the rest of the COFF runtime. +// +//===----------------------------------------------------------------------===// + +#define NOMINMAX +#include <windows.h> + +#include "coff_platform.h" + +#include "debug.h" +#include "error.h" +#include "wrapper_function_utils.h" + +#include <array> +#include <list> +#include <map> +#include <mutex> +#include <sstream> +#include <string_view> +#include <vector> + +#define DEBUG_TYPE "coff_platform" + +using namespace __orc_rt; + +namespace __orc_rt { + +using COFFJITDylibDepInfo = std::vector<ExecutorAddr>; +using COFFJITDylibDepInfoMap = + std::unordered_map<ExecutorAddr, COFFJITDylibDepInfo>; + +using SPSCOFFObjectSectionsMap = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>; + +using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>; + +using SPSCOFFJITDylibDepInfoMap = + SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>; + +} // namespace __orc_rt + +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_symbol_lookup_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_push_initializers_tag) + +namespace { +class COFFPlatformRuntimeState { +private: + // Ctor/dtor section. + // Manage lists of *tor functions sorted by the last character of subsection + // name. + struct XtorSection { + void Register(char SubsectionChar, span<void (*)(void)> Xtors) { + Subsections[SubsectionChar - 'A'].push_back(Xtors); + SubsectionsNew[SubsectionChar - 'A'].push_back(Xtors); + } + + void RegisterNoRun(char SubsectionChar, span<void (*)(void)> Xtors) { + Subsections[SubsectionChar - 'A'].push_back(Xtors); + } + + void Reset() { SubsectionsNew = Subsections; } + + void RunAllNewAndFlush(); + + private: + std::array<std::vector<span<void (*)(void)>>, 26> Subsections; + std::array<std::vector<span<void (*)(void)>>, 26> SubsectionsNew; + }; + + struct JITDylibState { + std::string Name; + void *Header = nullptr; + size_t LinkedAgainstRefCount = 0; + size_t DlRefCount = 0; + std::vector<JITDylibState *> Deps; + std::vector<void (*)(void)> AtExits; + XtorSection CInitSection; // XIA~XIZ + XtorSection CXXInitSection; // XCA~XCZ + XtorSection CPreTermSection; // XPA~XPZ + XtorSection CTermSection; // XTA~XTZ + + bool referenced() const { + return LinkedAgainstRefCount != 0 || DlRefCount != 0; + } + }; + +public: + static void initialize(); + static COFFPlatformRuntimeState &get(); + static bool isInitialized() { return CPS; } + static void destroy(); + + COFFPlatformRuntimeState() = default; + + // Delete copy and move constructors. + COFFPlatformRuntimeState(const COFFPlatformRuntimeState &) = delete; + COFFPlatformRuntimeState & + operator=(const COFFPlatformRuntimeState &) = delete; + COFFPlatformRuntimeState(COFFPlatformRuntimeState &&) = delete; + COFFPlatformRuntimeState &operator=(COFFPlatformRuntimeState &&) = delete; + + const char *dlerror(); + void *dlopen(std::string_view Name, int Mode); + int dlclose(void *Header); + void *dlsym(void *Header, std::string_view Symbol); + + Error registerJITDylib(std::string Name, void *Header); + Error deregisterJITDylib(void *Header); + + Error registerAtExit(ExecutorAddr HeaderAddr, void (*AtExit)(void)); + + Error registerObjectSections( + ExecutorAddr HeaderAddr, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs, + bool RunInitializers); + Error deregisterObjectSections( + ExecutorAddr HeaderAddr, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs); + + void *findJITDylibBaseByPC(uint64_t PC); + +private: + Error registerBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range); + Error deregisterBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range); + + Error registerSEHFrames(ExecutorAddr HeaderAddr, + ExecutorAddrRange SEHFrameRange); + Error deregisterSEHFrames(ExecutorAddr HeaderAddr, + ExecutorAddrRange SEHFrameRange); + + Expected<void *> dlopenImpl(std::string_view Path, int Mode); + Error dlopenFull(JITDylibState &JDS); + Error dlopenInitialize(JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo); + + Error dlcloseImpl(void *DSOHandle); + Error dlcloseDeinitialize(JITDylibState &JDS); + + JITDylibState *getJITDylibStateByHeader(void *DSOHandle); + JITDylibState *getJITDylibStateByName(std::string_view Path); + Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle, + std::string_view Symbol); + + static COFFPlatformRuntimeState *CPS; + + std::recursive_mutex JDStatesMutex; + std::map<void *, JITDylibState> JDStates; + struct BlockRange { + void *Header; + size_t Size; + }; + std::map<void *, BlockRange> BlockRanges; + std::unordered_map<std::string_view, void *> JDNameToHeader; + std::string DLFcnError; +}; + +} // namespace + +COFFPlatformRuntimeState *COFFPlatformRuntimeState::CPS = nullptr; + +COFFPlatformRuntimeState::JITDylibState * +COFFPlatformRuntimeState::getJITDylibStateByHeader(void *Header) { + auto I = JDStates.find(Header); + if (I == JDStates.end()) + return nullptr; + return &I->second; +} + +COFFPlatformRuntimeState::JITDylibState * +COFFPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { + // FIXME: Avoid creating string copy here. + auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); + if (I == JDNameToHeader.end()) + return nullptr; + void *H = I->second; + auto J = JDStates.find(H); + assert(J != JDStates.end() && + "JITDylib has name map entry but no header map entry"); + return &J->second; +} + +Error COFFPlatformRuntimeState::registerJITDylib(std::string Name, + void *Header) { + ORC_RT_DEBUG({ + printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header); + }); + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + if (JDStates.count(Header)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (name = " << Name << ")"; + return make_error<StringError>(ErrStream.str()); + } + if (JDNameToHeader.count(Name)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (header = " << Header << ")"; + return make_error<StringError>(ErrStream.str()); + } + + auto &JDS = JDStates[Header]; + JDS.Name = std::move(Name); + JDS.Header = Header; + JDNameToHeader[JDS.Name] = Header; + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterJITDylib(void *Header) { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto I = JDStates.find(Header); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Attempted to deregister unrecognized header " << Header; + return make_error<StringError>(ErrStream.str()); + } + + // Remove std::string construction once we can use C++20. + auto J = JDNameToHeader.find( + std::string(I->second.Name.data(), I->second.Name.size())); + assert(J != JDNameToHeader.end() && + "Missing JDNameToHeader entry for JITDylib"); + + ORC_RT_DEBUG({ + printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(), + Header); + }); + + JDNameToHeader.erase(J); + JDStates.erase(I); + return Error::success(); +} + +void COFFPlatformRuntimeState::XtorSection::RunAllNewAndFlush() { + for (auto &Subsection : SubsectionsNew) { + for (auto &XtorGroup : Subsection) + for (auto &Xtor : XtorGroup) + if (Xtor) + Xtor(); + Subsection.clear(); + } +} + +const char *COFFPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } + +void *COFFPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { + ORC_RT_DEBUG({ + std::string S(Path.data(), Path.size()); + printdbg("COFFPlatform::dlopen(\"%s\")\n", S.c_str()); + }); + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + if (auto H = dlopenImpl(Path, Mode)) + return *H; + else { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(H.takeError()); + return nullptr; + } +} + +int COFFPlatformRuntimeState::dlclose(void *DSOHandle) { + ORC_RT_DEBUG({ + auto *JDS = getJITDylibStateByHeader(DSOHandle); + std::string DylibName; + if (JDS) { + std::string S; + printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str()); + } else + printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, "invalid handle"); + }); + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + if (auto Err = dlcloseImpl(DSOHandle)) { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(std::move(Err)); + return -1; + } + return 0; +} + +void *COFFPlatformRuntimeState::dlsym(void *Header, std::string_view Symbol) { + auto Addr = lookupSymbolInJITDylib(Header, Symbol); + if (!Addr) { + return 0; + } + + return Addr->toPtr<void *>(); +} + +Expected<void *> COFFPlatformRuntimeState::dlopenImpl(std::string_view Path, + int Mode) { + // Try to find JITDylib state by name. + auto *JDS = getJITDylibStateByName(Path); + + if (!JDS) + return make_error<StringError>("No registered JTIDylib for path " + + std::string(Path.data(), Path.size())); + + if (auto Err = dlopenFull(*JDS)) + return std::move(Err); + + // Bump the ref-count on this dylib. + ++JDS->DlRefCount; + + // Return the header address. + return JDS->Header; +} + +Error COFFPlatformRuntimeState::dlopenFull(JITDylibState &JDS) { + // Call back to the JIT to push the initializers. + Expected<COFFJITDylibDepInfoMap> DepInfoMap((COFFJITDylibDepInfoMap())); + if (auto Err = WrapperFunction<SPSExpected<SPSCOFFJITDylibDepInfoMap>( + SPSExecutorAddr)>::call(&__orc_rt_coff_push_initializers_tag, + DepInfoMap, + ExecutorAddr::fromPtr(JDS.Header))) + return Err; + if (!DepInfoMap) + return DepInfoMap.takeError(); + + if (auto Err = dlopenInitialize(JDS, *DepInfoMap)) + return Err; + + if (!DepInfoMap->empty()) { + ORC_RT_DEBUG({ + printdbg("Unrecognized dep-info key headers in dlopen of %s\n", + JDS.Name.c_str()); + }); + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep-info key headers " + "while processing dlopen of " + << JDS.Name; + return make_error<StringError>(ErrStream.str()); + } + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlopenInitialize( + JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo) { + ORC_RT_DEBUG({ + printdbg("COFFPlatformRuntimeState::dlopenInitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + // Skip visited dependency. + auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header)); + if (I == DepInfo.end()) + return Error::success(); + + auto DI = std::move(I->second); + DepInfo.erase(I); + + // Run initializers of dependencies in proper order by depth-first traversal + // of dependency graph. + std::vector<JITDylibState *> OldDeps; + std::swap(JDS.Deps, OldDeps); + JDS.Deps.reserve(DI.size()); + for (auto DepHeaderAddr : DI) { + auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>()); + if (!DepJDS) { + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep header " + << DepHeaderAddr.toPtr<void *>() << " while initializing " + << JDS.Name; + return make_error<StringError>(ErrStream.str()); + } + ++DepJDS->LinkedAgainstRefCount; + if (auto Err = dlopenInitialize(*DepJDS, DepInfo)) + return Err; + } + + // Run static initializers. + JDS.CInitSection.RunAllNewAndFlush(); + JDS.CXXInitSection.RunAllNewAndFlush(); + + // Decrement old deps. + for (auto *DepJDS : OldDeps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(*DepJDS)) + return Err; + } + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { + // Try to find JITDylib state by header. + auto *JDS = getJITDylibStateByHeader(DSOHandle); + + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "No registered JITDylib for " << DSOHandle; + return make_error<StringError>(ErrStream.str()); + } + + // Bump the ref-count. + --JDS->DlRefCount; + + if (!JDS->referenced()) + return dlcloseDeinitialize(*JDS); + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) { + ORC_RT_DEBUG({ + printdbg("COFFPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + // Run atexits + for (auto AtExit : JDS.AtExits) + AtExit(); + JDS.AtExits.clear(); + + // Run static terminators. + JDS.CPreTermSection.RunAllNewAndFlush(); + JDS.CTermSection.RunAllNewAndFlush(); + + // Queue all xtors as new again. + JDS.CInitSection.Reset(); + JDS.CXXInitSection.Reset(); + JDS.CPreTermSection.Reset(); + JDS.CTermSection.Reset(); + + // Deinitialize any dependencies. + for (auto *DepJDS : JDS.Deps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(*DepJDS)) + return Err; + } + + return Error::success(); +} + +Expected<ExecutorAddr> +COFFPlatformRuntimeState::lookupSymbolInJITDylib(void *header, + std::string_view Sym) { + Expected<ExecutorAddr> Result((ExecutorAddr())); + if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>( + SPSExecutorAddr, SPSString)>::call(&__orc_rt_coff_symbol_lookup_tag, + Result, + ExecutorAddr::fromPtr(header), + Sym)) + return std::move(Err); + return Result; +} + +Error COFFPlatformRuntimeState::registerObjectSections( + ExecutorAddr HeaderAddr, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs, + bool RunInitializers) { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto I = JDStates.find(HeaderAddr.toPtr<void *>()); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Unrecognized header " << HeaderAddr.getValue(); + return make_error<StringError>(ErrStream.str()); + } + auto &JDState = I->second; + for (auto &KV : Secs) { + if (auto Err = registerBlockRange(HeaderAddr, KV.second)) + return Err; + if (KV.first.empty()) + continue; + char LastChar = KV.first.data()[KV.first.size() - 1]; + if (KV.first == ".pdata") { + if (auto Err = registerSEHFrames(HeaderAddr, KV.second)) + return Err; + } else if (KV.first >= ".CRT$XIA" && KV.first <= ".CRT$XIZ") { + if (RunInitializers) + JDState.CInitSection.Register(LastChar, + KV.second.toSpan<void (*)(void)>()); + else + JDState.CInitSection.RegisterNoRun(LastChar, + KV.second.toSpan<void (*)(void)>()); + } else if (KV.first >= ".CRT$XCA" && KV.first <= ".CRT$XCZ") { + if (RunInitializers) + JDState.CXXInitSection.Register(LastChar, + KV.second.toSpan<void (*)(void)>()); + else + JDState.CXXInitSection.RegisterNoRun( + LastChar, KV.second.toSpan<void (*)(void)>()); + } else if (KV.first >= ".CRT$XPA" && KV.first <= ".CRT$XPZ") + JDState.CPreTermSection.Register(LastChar, + KV.second.toSpan<void (*)(void)>()); + else if (KV.first >= ".CRT$XTA" && KV.first <= ".CRT$XTZ") + JDState.CTermSection.Register(LastChar, + KV.second.toSpan<void (*)(void)>()); + } + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterObjectSections( + ExecutorAddr HeaderAddr, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto I = JDStates.find(HeaderAddr.toPtr<void *>()); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Attempted to deregister unrecognized header " + << HeaderAddr.getValue(); + return make_error<StringError>(ErrStream.str()); + } + for (auto &KV : Secs) { + if (auto Err = deregisterBlockRange(HeaderAddr, KV.second)) + return Err; + if (KV.first == ".pdata") + if (auto Err = deregisterSEHFrames(HeaderAddr, KV.second)) + return Err; + } + return Error::success(); +} + +Error COFFPlatformRuntimeState::registerSEHFrames( + ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) { + int N = (SEHFrameRange.End.getValue() - SEHFrameRange.Start.getValue()) / + sizeof(RUNTIME_FUNCTION); + auto Func = SEHFrameRange.Start.toPtr<PRUNTIME_FUNCTION>(); + if (!RtlAddFunctionTable(Func, N, + static_cast<DWORD64>(HeaderAddr.getValue()))) + return make_error<StringError>("Failed to register SEH frames"); + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterSEHFrames( + ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) { + if (!RtlDeleteFunctionTable(SEHFrameRange.Start.toPtr<PRUNTIME_FUNCTION>())) + return make_error<StringError>("Failed to deregister SEH frames"); + return Error::success(); +} + +Error COFFPlatformRuntimeState::registerBlockRange(ExecutorAddr HeaderAddr, + ExecutorAddrRange Range) { + assert(!BlockRanges.count(Range.Start.toPtr<void *>()) && + "Block range address already registered"); + BlockRange B = {HeaderAddr.toPtr<void *>(), Range.size()}; + BlockRanges.emplace(Range.Start.toPtr<void *>(), B); + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterBlockRange(ExecutorAddr HeaderAddr, + ExecutorAddrRange Range) { + assert(BlockRanges.count(Range.Start.toPtr<void *>()) && + "Block range address not registered"); + BlockRanges.erase(Range.Start.toPtr<void *>()); + return Error::success(); +} + +Error COFFPlatformRuntimeState::registerAtExit(ExecutorAddr HeaderAddr, + void (*AtExit)(void)) { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto I = JDStates.find(HeaderAddr.toPtr<void *>()); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Unrecognized header " << HeaderAddr.getValue(); + return make_error<StringError>(ErrStream.str()); + } + I->second.AtExits.push_back(AtExit); + return Error::success(); +} + +void COFFPlatformRuntimeState::initialize() { + assert(!CPS && "COFFPlatformRuntimeState should be null"); + CPS = new COFFPlatformRuntimeState(); +} + +COFFPlatformRuntimeState &COFFPlatformRuntimeState::get() { + assert(CPS && "COFFPlatformRuntimeState not initialized"); + return *CPS; +} + +void COFFPlatformRuntimeState::destroy() { + assert(CPS && "COFFPlatformRuntimeState not initialized"); + delete CPS; +} + +void *COFFPlatformRuntimeState::findJITDylibBaseByPC(uint64_t PC) { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto It = BlockRanges.upper_bound(reinterpret_cast<void *>(PC)); + if (It == BlockRanges.begin()) + return nullptr; + --It; + auto &Range = It->second; + if (PC >= reinterpret_cast<uint64_t>(It->first) + Range.Size) + return nullptr; + return Range.Header; +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_coff_platform_bootstrap(char *ArgData, size_t ArgSize) { + COFFPlatformRuntimeState::initialize(); + return WrapperFunctionResult().release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_coff_platform_shutdown(char *ArgData, size_t ArgSize) { + COFFPlatformRuntimeState::destroy(); + return WrapperFunctionResult().release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_coff_register_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle( + ArgData, ArgSize, + [](std::string &Name, ExecutorAddr HeaderAddr) { + return COFFPlatformRuntimeState::get().registerJITDylib( + std::move(Name), HeaderAddr.toPtr<void *>()); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_coff_deregister_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSExecutorAddr)>::handle( + ArgData, ArgSize, + [](ExecutorAddr HeaderAddr) { + return COFFPlatformRuntimeState::get().deregisterJITDylib( + HeaderAddr.toPtr<void *>()); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_coff_register_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSExecutorAddr, SPSCOFFObjectSectionsMap, + bool)>:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> + &Secs, + bool RunInitializers) { + return COFFPlatformRuntimeState::get().registerObjectSections( + HeaderAddr, std::move(Secs), RunInitializers); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_coff_deregister_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSExecutorAddr, SPSCOFFObjectSectionsMap)>:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> + &Secs) { + return COFFPlatformRuntimeState::get().deregisterObjectSections( + HeaderAddr, std::move(Secs)); + }) + .release(); +} +//------------------------------------------------------------------------------ +// JIT'd dlfcn alternatives. +//------------------------------------------------------------------------------ + +const char *__orc_rt_coff_jit_dlerror() { + return COFFPlatformRuntimeState::get().dlerror(); +} + +void *__orc_rt_coff_jit_dlopen(const char *path, int mode) { + return COFFPlatformRuntimeState::get().dlopen(path, mode); +} + +int __orc_rt_coff_jit_dlclose(void *header) { + return COFFPlatformRuntimeState::get().dlclose(header); +} + +void *__orc_rt_coff_jit_dlsym(void *header, const char *symbol) { + return COFFPlatformRuntimeState::get().dlsym(header, symbol); +} + +//------------------------------------------------------------------------------ +// COFF SEH exception support +//------------------------------------------------------------------------------ + +struct ThrowInfo { + uint32_t attributes; + void *data; +}; + +ORC_RT_INTERFACE void __stdcall __orc_rt_coff_cxx_throw_exception( + void *pExceptionObject, ThrowInfo *pThrowInfo) { +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmultichar" +#endif + constexpr uint32_t EH_EXCEPTION_NUMBER = 'msc' | 0xE0000000; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + constexpr uint32_t EH_MAGIC_NUMBER1 = 0x19930520; + auto BaseAddr = COFFPlatformRuntimeState::get().findJITDylibBaseByPC( + reinterpret_cast<uint64_t>(pThrowInfo)); + if (!BaseAddr) { + // This is not from JIT'd region. + // FIXME: Use the default implementation like below when alias api is + // capable. _CxxThrowException(pExceptionObject, pThrowInfo); + fprintf(stderr, "Throwing exception from compiled callback into JIT'd " + "exception handler not supported yet.\n"); + abort(); + return; + } + const ULONG_PTR parameters[] = { + EH_MAGIC_NUMBER1, + reinterpret_cast<ULONG_PTR>(pExceptionObject), + reinterpret_cast<ULONG_PTR>(pThrowInfo), + reinterpret_cast<ULONG_PTR>(BaseAddr), + }; + RaiseException(EH_EXCEPTION_NUMBER, EXCEPTION_NONCONTINUABLE, + _countof(parameters), parameters); +} + +//------------------------------------------------------------------------------ +// COFF atexits +//------------------------------------------------------------------------------ + +typedef int (*OnExitFunction)(void); +typedef void (*AtExitFunction)(void); + +ORC_RT_INTERFACE OnExitFunction __orc_rt_coff_onexit(void *Header, + OnExitFunction Func) { + if (auto Err = COFFPlatformRuntimeState::get().registerAtExit( + ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) { + consumeError(std::move(Err)); + return nullptr; + } + return Func; +} + +ORC_RT_INTERFACE int __orc_rt_coff_atexit(void *Header, AtExitFunction Func) { + if (auto Err = COFFPlatformRuntimeState::get().registerAtExit( + ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) { + consumeError(std::move(Err)); + return -1; + } + return 0; +} + +//------------------------------------------------------------------------------ +// COFF Run Program +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE int64_t __orc_rt_coff_run_program(const char *JITDylibName, + const char *EntrySymbolName, + int argc, char *argv[]) { + using MainTy = int (*)(int, char *[]); + + void *H = + __orc_rt_coff_jit_dlopen(JITDylibName, __orc_rt::coff::ORC_RT_RTLD_LAZY); + if (!H) { + __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); + return -1; + } + + auto *Main = + reinterpret_cast<MainTy>(__orc_rt_coff_jit_dlsym(H, EntrySymbolName)); + + if (!Main) { + __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); + return -1; + } + + int Result = Main(argc, argv); + + if (__orc_rt_coff_jit_dlclose(H) == -1) + __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); + + return Result; +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.h b/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.h new file mode 100644 index 000000000000..c84185d40b60 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.h @@ -0,0 +1,39 @@ +//===- coff_platform.h -------------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC Runtime support for dynamic loading features on COFF-based platforms. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_COFF_PLATFORM_H +#define ORC_RT_COFF_PLATFORM_H + +#include "common.h" +#include "executor_address.h" + +// dlfcn functions. +ORC_RT_INTERFACE const char *__orc_rt_coff_jit_dlerror(); +ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlopen(const char *path, int mode); +ORC_RT_INTERFACE int __orc_rt_coff_jit_dlclose(void *header); +ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlsym(void *header, + const char *symbol); + +namespace __orc_rt { +namespace coff { + +enum dlopen_mode : int { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 +}; + +} // end namespace coff +} // end namespace __orc_rt + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.per_jd.cpp b/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.per_jd.cpp new file mode 100644 index 000000000000..6c208cb31858 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/coff_platform.per_jd.cpp @@ -0,0 +1,31 @@ +//===- coff_platform.per_jd.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code that will be loaded per each JITDylib. +// +//===----------------------------------------------------------------------===// +#include "compiler.h" + +ORC_RT_INTERFACE void __orc_rt_coff_per_jd_marker() {} + +typedef int (*OnExitFunction)(void); +typedef void (*AtExitFunction)(void); + +extern "C" void *__ImageBase; +ORC_RT_INTERFACE OnExitFunction __orc_rt_coff_onexit(void *Header, + OnExitFunction Func); +ORC_RT_INTERFACE int __orc_rt_coff_atexit(void *Header, AtExitFunction Func); + +ORC_RT_INTERFACE OnExitFunction +__orc_rt_coff_onexit_per_jd(OnExitFunction Func) { + return __orc_rt_coff_onexit(&__ImageBase, Func); +} + +ORC_RT_INTERFACE int __orc_rt_coff_atexit_per_jd(AtExitFunction Func) { + return __orc_rt_coff_atexit(&__ImageBase, Func); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/common.h b/contrib/llvm-project/compiler-rt/lib/orc/common.h new file mode 100644 index 000000000000..73c5c4a2bd8d --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/common.h @@ -0,0 +1,48 @@ +//===- common.h - Common utilities for the ORC runtime ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_COMMON_H +#define ORC_RT_COMMON_H + +#include "compiler.h" +#include "orc_rt/c_api.h" +#include <type_traits> + +/// This macro should be used to define tags that will be associated with +/// handlers in the JIT process, and call can be used to define tags f +#define ORC_RT_JIT_DISPATCH_TAG(X) \ +extern "C" char X; \ +char X = 0; + +/// Opaque struct for external symbols. +struct __orc_rt_Opaque {}; + +/// Error reporting function. +extern "C" void __orc_rt_log_error(const char *ErrMsg); + +/// Context object for dispatching calls to the JIT object. +/// +/// This is declared for use by the runtime, but should be implemented in the +/// executor or provided by a definition added to the JIT before the runtime +/// is loaded. +ORC_RT_IMPORT __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT; + +/// For dispatching calls to the JIT object. +/// +/// This is declared for use by the runtime, but should be implemented in the +/// executor or provided by a definition added to the JIT before the runtime +/// is loaded. +ORC_RT_IMPORT orc_rt_CWrapperFunctionResult +__orc_rt_jit_dispatch(__orc_rt_Opaque *DispatchCtx, const void *FnTag, + const char *Data, size_t Size) ORC_RT_WEAK_IMPORT; + +#endif // ORC_RT_COMMON_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/compiler.h b/contrib/llvm-project/compiler-rt/lib/orc/compiler.h new file mode 100644 index 000000000000..88cb3d92b03b --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/compiler.h @@ -0,0 +1,74 @@ +//===--------- compiler.h - Compiler abstraction support --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +// Most functionality in this file was swiped from llvm/Support/Compiler.h. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_COMPILER_H +#define ORC_RT_COMPILER_H + +#if defined(_WIN32) +#define ORC_RT_INTERFACE extern "C" +#define ORC_RT_HIDDEN +#define ORC_RT_IMPORT extern "C" __declspec(dllimport) +#else +#define ORC_RT_INTERFACE extern "C" __attribute__((visibility("default"))) +#define ORC_RT_HIDDEN __attribute__((visibility("hidden"))) +#define ORC_RT_IMPORT extern "C" +#endif + +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +// Only use __has_cpp_attribute in C++ mode. GCC defines __has_cpp_attribute in +// C mode, but the :: in __has_cpp_attribute(scoped::attribute) is invalid. +#ifndef ORC_RT_HAS_CPP_ATTRIBUTE +#if defined(__cplusplus) && defined(__has_cpp_attribute) +#define ORC_RT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +#define ORC_RT_HAS_CPP_ATTRIBUTE(x) 0 +#endif +#endif + +// Use the 'nodiscard' attribute in C++17 or newer mode. +#if defined(__cplusplus) && __cplusplus > 201402L && \ + ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard) +#define ORC_RT_NODISCARD [[nodiscard]] +#elif ORC_RT_HAS_CPP_ATTRIBUTE(clang::warn_unused_result) +#define ORC_RT_NODISCARD [[clang::warn_unused_result]] +// Clang in C++14 mode claims that it has the 'nodiscard' attribute, but also +// warns in the pedantic mode that 'nodiscard' is a C++17 extension (PR33518). +// Use the 'nodiscard' attribute in C++14 mode only with GCC. +// TODO: remove this workaround when PR33518 is resolved. +#elif defined(__GNUC__) && ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard) +#define ORC_RT_NODISCARD [[nodiscard]] +#else +#define ORC_RT_NODISCARD +#endif + +#if __has_builtin(__builtin_expect) +#define ORC_RT_LIKELY(EXPR) __builtin_expect((bool)(EXPR), true) +#define ORC_RT_UNLIKELY(EXPR) __builtin_expect((bool)(EXPR), false) +#else +#define ORC_RT_LIKELY(EXPR) (EXPR) +#define ORC_RT_UNLIKELY(EXPR) (EXPR) +#endif + +#if defined(__APPLE__) +#define ORC_RT_WEAK_IMPORT __attribute__((weak_import)) +#elif defined(_WIN32) +#define ORC_RT_WEAK_IMPORT +#else +#define ORC_RT_WEAK_IMPORT __attribute__((weak)) +#endif + +#endif // ORC_RT_COMPILER_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/debug.cpp b/contrib/llvm-project/compiler-rt/lib/orc/debug.cpp new file mode 100644 index 000000000000..af20fa4e6f4e --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/debug.cpp @@ -0,0 +1,83 @@ +//===- debug.cpp ----------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "debug.h" + +#include <cassert> +#include <cstdarg> +#include <cstdio> +#include <cstdlib> +#include <cstring> + + +namespace __orc_rt { + +#ifndef NDEBUG + +std::atomic<const char *> DebugTypes; +char DebugTypesAll; +char DebugTypesNone; + +/// Sets the DebugState and DebugTypes values -- this function may be called +/// concurrently on multiple threads, but will always assign the same values so +/// this should be safe. +const char *initializeDebug() { + if (const char *DT = getenv("ORC_RT_DEBUG")) { + // If ORC_RT_DEBUG=1 then log everything. + if (strcmp(DT, "1") == 0) { + DebugTypes.store(&DebugTypesAll, std::memory_order_relaxed); + return &DebugTypesAll; + } + + // If ORC_RT_DEBUG is non-empty then record the string for use in + // debugTypeEnabled. + if (strcmp(DT, "") != 0) { + DebugTypes.store(DT, std::memory_order_relaxed); + return DT; + } + } + + // If ORT_RT_DEBUG is undefined or defined as empty then log nothing. + DebugTypes.store(&DebugTypesNone, std::memory_order_relaxed); + return &DebugTypesNone; +} + +bool debugTypeEnabled(const char *Type, const char *Types) { + assert(Types && Types != &DebugTypesAll && Types != &DebugTypesNone && + "Invalid Types value"); + size_t TypeLen = strlen(Type); + const char *Start = Types; + const char *End = Start; + + do { + if (*End == '\0' || *End == ',') { + size_t ItemLen = End - Start; + if (ItemLen == TypeLen && memcmp(Type, Start, TypeLen) == 0) + return true; + if (*End == '\0') + return false; + Start = End + 1; + } + ++End; + } while (true); +} + +void printdbg(const char *format, ...) { + va_list Args; + va_start(Args, format); + vfprintf(stderr, format, Args); + va_end(Args); +} + +#endif // !NDEBUG + +} // end namespace __orc_rt diff --git a/contrib/llvm-project/compiler-rt/lib/orc/debug.h b/contrib/llvm-project/compiler-rt/lib/orc/debug.h new file mode 100644 index 000000000000..a0bc653d032e --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/debug.h @@ -0,0 +1,56 @@ +//===- debug.h - Debugging output utilities ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_DEBUG_H +#define ORC_RT_DEBUG_H + +#include <atomic> + +#ifndef NDEBUG + +namespace __orc_rt { + +extern std::atomic<const char *> DebugTypes; +extern char DebugTypesAll; +extern char DebugTypesNone; + +const char *initializeDebug(); +bool debugTypeEnabled(const char *Type, const char *Types); +void printdbg(const char *format, ...); + +} // namespace __orc_rt + +#define ORC_RT_DEBUG_WITH_TYPE(TYPE, X) \ + do { \ + const char *Types = \ + ::__orc_rt::DebugTypes.load(std::memory_order_relaxed); \ + if (!Types) \ + Types = initializeDebug(); \ + if (Types == &DebugTypesNone) \ + break; \ + if (Types == &DebugTypesAll || \ + ::__orc_rt::debugTypeEnabled(TYPE, Types)) { \ + X; \ + } \ + } while (false) + +#else + +#define ORC_RT_DEBUG_WITH_TYPE(TYPE, X) \ + do { \ + } while (false) + +#endif // !NDEBUG + +#define ORC_RT_DEBUG(X) ORC_RT_DEBUG_WITH_TYPE(DEBUG_TYPE, X) + +#endif // ORC_RT_DEBUG_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/dlfcn_wrapper.cpp b/contrib/llvm-project/compiler-rt/lib/orc/dlfcn_wrapper.cpp new file mode 100644 index 000000000000..ece63da2cb48 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/dlfcn_wrapper.cpp @@ -0,0 +1,52 @@ +//===- dlfcn_wrapper.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "adt.h" +#include "common.h" +#include "wrapper_function_utils.h" + +#include <vector> + +using namespace __orc_rt; + +extern "C" const char *__orc_rt_jit_dlerror(); +extern "C" void *__orc_rt_jit_dlopen(const char *path, int mode); +extern "C" int __orc_rt_jit_dlclose(void *dso_handle); + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_jit_dlerror_wrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSString()>::handle( + ArgData, ArgSize, + []() { return std::string(__orc_rt_jit_dlerror()); }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_jit_dlopen_wrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSExecutorAddr(SPSString, int32_t)>::handle( + ArgData, ArgSize, + [](const std::string &Path, int32_t mode) { + return ExecutorAddr::fromPtr( + __orc_rt_jit_dlopen(Path.c_str(), mode)); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_jit_dlclose_wrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<int32_t(SPSExecutorAddr)>::handle( + ArgData, ArgSize, + [](ExecutorAddr &DSOHandle) { + return __orc_rt_jit_dlclose(DSOHandle.toPtr<void *>()); + }) + .release(); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/elfnix_platform.cpp b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_platform.cpp new file mode 100644 index 000000000000..c087e71038f9 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_platform.cpp @@ -0,0 +1,624 @@ +//===- elfnix_platform.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code required to load the rest of the ELF-on-*IX runtime. +// +//===----------------------------------------------------------------------===// + +#include "elfnix_platform.h" +#include "common.h" +#include "compiler.h" +#include "error.h" +#include "wrapper_function_utils.h" + +#include <algorithm> +#include <map> +#include <mutex> +#include <sstream> +#include <string_view> +#include <unordered_map> +#include <vector> + +using namespace __orc_rt; +using namespace __orc_rt::elfnix; + +// Declare function tags for functions in the JIT process. +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag) + +// eh-frame registration functions, made available via aliases +// installed by the Platform +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +extern "C" void +__unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT; +extern "C" void +__unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT; + +namespace { + +Error validatePointerSectionExtent(const char *SectionName, + const ExecutorAddrRange &SE) { + if (SE.size() % sizeof(uintptr_t)) { + std::ostringstream ErrMsg; + ErrMsg << std::hex << "Size of " << SectionName << " 0x" + << SE.Start.getValue() << " -- 0x" << SE.End.getValue() + << " is not a pointer multiple"; + return make_error<StringError>(ErrMsg.str()); + } + return Error::success(); +} + +Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections, + const ELFNixJITDylibInitializers &MOJDIs) { + + for (const auto &ModInits : InitArraySections) { + if (auto Err = validatePointerSectionExtent(".init_array", ModInits)) + return Err; + + using InitFunc = void (*)(); + for (auto *Init : ModInits.toSpan<InitFunc>()) + (*Init)(); + } + + return Error::success(); +} + +struct TLSInfoEntry { + unsigned long Key = 0; + unsigned long DataAddress = 0; +}; + +struct TLSDescriptor { + void (*Resolver)(void *); + TLSInfoEntry *InfoEntry; +}; + +class ELFNixPlatformRuntimeState { +private: + struct AtExitEntry { + void (*Func)(void *); + void *Arg; + }; + + using AtExitsVector = std::vector<AtExitEntry>; + + struct PerJITDylibState { + void *Header = nullptr; + size_t RefCount = 0; + bool AllowReinitialization = false; + AtExitsVector AtExits; + }; + +public: + static void initialize(void *DSOHandle); + static ELFNixPlatformRuntimeState &get(); + static void destroy(); + + ELFNixPlatformRuntimeState(void *DSOHandle); + + // Delete copy and move constructors. + ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete; + ELFNixPlatformRuntimeState & + operator=(const ELFNixPlatformRuntimeState &) = delete; + ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete; + ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete; + + Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR); + Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR); + + const char *dlerror(); + void *dlopen(std::string_view Name, int Mode); + int dlclose(void *DSOHandle); + void *dlsym(void *DSOHandle, std::string_view Symbol); + + int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); + void runAtExits(void *DSOHandle); + + /// Returns the base address of the section containing ThreadData. + Expected<std::pair<const char *, size_t>> + getThreadDataSectionFor(const char *ThreadData); + + void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; } + +private: + PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); + PerJITDylibState *getJITDylibStateByName(std::string_view Path); + PerJITDylibState & + getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs); + + Error registerThreadDataSection(span<const char> ThreadDataSection); + + Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle, + std::string_view Symbol); + + Expected<ELFNixJITDylibInitializerSequence> + getJITDylibInitializersByName(std::string_view Path); + Expected<void *> dlopenInitialize(std::string_view Path, int Mode); + Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs); + + static ELFNixPlatformRuntimeState *MOPS; + + void *PlatformJDDSOHandle; + + // Frame registration functions: + void (*registerEHFrameSection)(const void *) = nullptr; + void (*deregisterEHFrameSection)(const void *) = nullptr; + + // FIXME: Move to thread-state. + std::string DLFcnError; + + std::recursive_mutex JDStatesMutex; + std::unordered_map<void *, PerJITDylibState> JDStates; + std::unordered_map<std::string, void *> JDNameToHeader; + + std::mutex ThreadDataSectionsMutex; + std::map<const char *, size_t> ThreadDataSections; +}; + +ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr; + +void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) { + assert(!MOPS && "ELFNixPlatformRuntimeState should be null"); + MOPS = new ELFNixPlatformRuntimeState(DSOHandle); +} + +ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() { + assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); + return *MOPS; +} + +void ELFNixPlatformRuntimeState::destroy() { + assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); + delete MOPS; +} + +ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle) + : PlatformJDDSOHandle(DSOHandle) { + if (__unw_add_dynamic_eh_frame_section && + __unw_remove_dynamic_eh_frame_section) { + registerEHFrameSection = __unw_add_dynamic_eh_frame_section; + deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section; + } else { + registerEHFrameSection = __register_frame; + deregisterEHFrameSection = __deregister_frame; + } +} + +Error ELFNixPlatformRuntimeState::registerObjectSections( + ELFNixPerObjectSectionsToRegister POSR) { + if (POSR.EHFrameSection.Start) + registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>()); + + if (POSR.ThreadDataSection.Start) { + if (auto Err = registerThreadDataSection( + POSR.ThreadDataSection.toSpan<const char>())) + return Err; + } + + return Error::success(); +} + +Error ELFNixPlatformRuntimeState::deregisterObjectSections( + ELFNixPerObjectSectionsToRegister POSR) { + if (POSR.EHFrameSection.Start) + deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>()); + + return Error::success(); +} + +const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } + +void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + + // Use fast path if all JITDylibs are already loaded and don't require + // re-running initializers. + if (auto *JDS = getJITDylibStateByName(Path)) { + if (!JDS->AllowReinitialization) { + ++JDS->RefCount; + return JDS->Header; + } + } + + auto H = dlopenInitialize(Path, Mode); + if (!H) { + DLFcnError = toString(H.takeError()); + return nullptr; + } + + return *H; +} + +int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) { + runAtExits(DSOHandle); + return 0; +} + +void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, + std::string_view Symbol) { + auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); + if (!Addr) { + DLFcnError = toString(Addr.takeError()); + return 0; + } + + return Addr->toPtr<void *>(); +} + +int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, + void *DSOHandle) { + // FIXME: Handle out-of-memory errors, returning -1 if OOM. + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); + assert(JDS && "JITDylib state not initialized"); + JDS->AtExits.push_back({F, Arg}); + return 0; +} + +void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) { + // FIXME: Should atexits be allowed to run concurrently with access to + // JDState? + AtExitsVector V; + { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); + assert(JDS && "JITDlybi state not initialized"); + std::swap(V, JDS->AtExits); + } + + while (!V.empty()) { + auto &AE = V.back(); + AE.Func(AE.Arg); + V.pop_back(); + } +} + +Expected<std::pair<const char *, size_t>> +ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { + std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadData); + // Check that we have a valid entry conovering this address. + if (I == ThreadDataSections.begin()) + return make_error<StringError>("No thread local data section for key"); + I = std::prev(I); + if (ThreadData >= I->first + I->second) + return make_error<StringError>("No thread local data section for key"); + return *I; +} + +ELFNixPlatformRuntimeState::PerJITDylibState * +ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { + auto I = JDStates.find(DSOHandle); + if (I == JDStates.end()) + return nullptr; + return &I->second; +} + +ELFNixPlatformRuntimeState::PerJITDylibState * +ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { + // FIXME: Avoid creating string copy here. + auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); + if (I == JDNameToHeader.end()) + return nullptr; + void *H = I->second; + auto J = JDStates.find(H); + assert(J != JDStates.end() && + "JITDylib has name map entry but no header map entry"); + return &J->second; +} + +ELFNixPlatformRuntimeState::PerJITDylibState & +ELFNixPlatformRuntimeState::getOrCreateJITDylibState( + ELFNixJITDylibInitializers &MOJDIs) { + void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>(); + + auto &JDS = JDStates[Header]; + + // If this entry hasn't been created yet. + if (!JDS.Header) { + assert(!JDNameToHeader.count(MOJDIs.Name) && + "JITDylib has header map entry but no name map entry"); + JDNameToHeader[MOJDIs.Name] = Header; + JDS.Header = Header; + } + + return JDS; +} + +Error ELFNixPlatformRuntimeState::registerThreadDataSection( + span<const char> ThreadDataSection) { + std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); + if (I != ThreadDataSections.begin()) { + auto J = std::prev(I); + if (J->first + J->second > ThreadDataSection.data()) + return make_error<StringError>("Overlapping .tdata sections"); + } + ThreadDataSections.insert( + I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); + return Error::success(); +} + +Expected<ExecutorAddr> +ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, + std::string_view Sym) { + Expected<ExecutorAddr> Result((ExecutorAddr())); + if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>( + SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag, + Result, + ExecutorAddr::fromPtr(DSOHandle), + Sym)) + return std::move(Err); + return Result; +} + +Expected<ELFNixJITDylibInitializerSequence> +ELFNixPlatformRuntimeState::getJITDylibInitializersByName( + std::string_view Path) { + Expected<ELFNixJITDylibInitializerSequence> Result( + (ELFNixJITDylibInitializerSequence())); + std::string PathStr(Path.data(), Path.size()); + if (auto Err = + WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>( + SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result, + Path)) + return std::move(Err); + return Result; +} + +Expected<void *> +ELFNixPlatformRuntimeState::dlopenInitialize(std::string_view Path, int Mode) { + // Either our JITDylib wasn't loaded, or it or one of its dependencies allows + // reinitialization. We need to call in to the JIT to see if there's any new + // work pending. + auto InitSeq = getJITDylibInitializersByName(Path); + if (!InitSeq) + return InitSeq.takeError(); + + // Init sequences should be non-empty. + if (InitSeq->empty()) + return make_error<StringError>( + "__orc_rt_elfnix_get_initializers returned an " + "empty init sequence"); + + // Otherwise register and run initializers for each JITDylib. + for (auto &MOJDIs : *InitSeq) + if (auto Err = initializeJITDylib(MOJDIs)) + return std::move(Err); + + // Return the header for the last item in the list. + auto *JDS = getJITDylibStateByHeaderAddr( + InitSeq->back().DSOHandleAddress.toPtr<void *>()); + assert(JDS && "Missing state entry for JD"); + return JDS->Header; +} + +long getPriority(const std::string &name) { + auto pos = name.find_last_not_of("0123456789"); + if (pos == name.size() - 1) + return 65535; + else + return std::strtol(name.c_str() + pos + 1, nullptr, 10); +} + +Error ELFNixPlatformRuntimeState::initializeJITDylib( + ELFNixJITDylibInitializers &MOJDIs) { + + auto &JDS = getOrCreateJITDylibState(MOJDIs); + ++JDS.RefCount; + + using SectionList = std::vector<ExecutorAddrRange>; + std::sort(MOJDIs.InitSections.begin(), MOJDIs.InitSections.end(), + [](const std::pair<std::string, SectionList> &LHS, + const std::pair<std::string, SectionList> &RHS) -> bool { + return getPriority(LHS.first) < getPriority(RHS.first); + }); + for (auto &Entry : MOJDIs.InitSections) + if (auto Err = runInitArray(Entry.second, MOJDIs)) + return Err; + + return Error::success(); +} +class ELFNixPlatformRuntimeTLVManager { +public: + void *getInstance(const char *ThreadData); + +private: + std::unordered_map<const char *, char *> Instances; + std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; +}; + +void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { + auto I = Instances.find(ThreadData); + if (I != Instances.end()) + return I->second; + auto TDS = + ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); + if (!TDS) { + __orc_rt_log_error(toString(TDS.takeError()).c_str()); + return nullptr; + } + + auto &Allocated = AllocatedSections[TDS->first]; + if (!Allocated) { + Allocated = std::make_unique<char[]>(TDS->second); + memcpy(Allocated.get(), TDS->first, TDS->second); + } + size_t ThreadDataDelta = ThreadData - TDS->first; + assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); + + char *Instance = Allocated.get() + ThreadDataDelta; + Instances[ThreadData] = Instance; + return Instance; +} + +void destroyELFNixTLVMgr(void *ELFNixTLVMgr) { + delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr); +} + +} // end anonymous namespace + +//------------------------------------------------------------------------------ +// JIT entry points +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) { + return WrapperFunction<void(uint64_t)>::handle( + ArgData, ArgSize, + [](uint64_t &DSOHandle) { + ELFNixPlatformRuntimeState::initialize( + reinterpret_cast<void *>(DSOHandle)); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) { + ELFNixPlatformRuntimeState::destroy(); + return WrapperFunctionResult().release(); +} + +/// Wrapper function for registering metadata on a per-object basis. +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: + handle(ArgData, ArgSize, + [](ELFNixPerObjectSectionsToRegister &POSR) { + return ELFNixPlatformRuntimeState::get().registerObjectSections( + std::move(POSR)); + }) + .release(); +} + +/// Wrapper for releasing per-object metadat. +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: + handle(ArgData, ArgSize, + [](ELFNixPerObjectSectionsToRegister &POSR) { + return ELFNixPlatformRuntimeState::get() + .deregisterObjectSections(std::move(POSR)); + }) + .release(); +} + +//------------------------------------------------------------------------------ +// TLV support +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) { + auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>( + pthread_getspecific(D->Key)); + if (!TLVMgr) + TLVMgr = new ELFNixPlatformRuntimeTLVManager(); + if (pthread_setspecific(D->Key, TLVMgr)) { + __orc_rt_log_error("Call to pthread_setspecific failed"); + return nullptr; + } + + return TLVMgr->getInstance( + reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); +} + +ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl( + TLSDescriptor *D, const char *ThreadPointer) { + const char *TLVPtr = reinterpret_cast<const char *>( + __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry)); + return TLVPtr - ThreadPointer; +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( + ArgData, ArgSize, + []() -> Expected<uint64_t> { + pthread_key_t Key; + if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) { + __orc_rt_log_error("Call to pthread_key_create failed"); + return make_error<StringError>(strerror(Err)); + } + return static_cast<uint64_t>(Key); + }) + .release(); +} + +//------------------------------------------------------------------------------ +// cxa_atexit support +//------------------------------------------------------------------------------ + +int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle) { + return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg, + dso_handle); +} + +int __orc_rt_elfnix_atexit(void (*func)(void *)) { + auto &PlatformRTState = ELFNixPlatformRuntimeState::get(); + return ELFNixPlatformRuntimeState::get().registerAtExit( + func, NULL, PlatformRTState.getPlatformJDDSOHandle()); +} + +void __orc_rt_elfnix_cxa_finalize(void *dso_handle) { + ELFNixPlatformRuntimeState::get().runAtExits(dso_handle); +} + +//------------------------------------------------------------------------------ +// JIT'd dlfcn alternatives. +//------------------------------------------------------------------------------ + +const char *__orc_rt_elfnix_jit_dlerror() { + return ELFNixPlatformRuntimeState::get().dlerror(); +} + +void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) { + return ELFNixPlatformRuntimeState::get().dlopen(path, mode); +} + +int __orc_rt_elfnix_jit_dlclose(void *dso_handle) { + return ELFNixPlatformRuntimeState::get().dlclose(dso_handle); +} + +void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) { + return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol); +} + +//------------------------------------------------------------------------------ +// ELFNix Run Program +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program( + const char *JITDylibName, const char *EntrySymbolName, int argc, + char *argv[]) { + using MainTy = int (*)(int, char *[]); + + void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName, + __orc_rt::elfnix::ORC_RT_RTLD_LAZY); + if (!H) { + __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); + return -1; + } + + auto *Main = + reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName)); + + if (!Main) { + __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); + return -1; + } + + int Result = Main(argc, argv); + + if (__orc_rt_elfnix_jit_dlclose(H) == -1) + __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); + + return Result; +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/elfnix_platform.h b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_platform.h new file mode 100644 index 000000000000..e0ee9591dfc6 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_platform.h @@ -0,0 +1,131 @@ +//===- elfnix_platform.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC Runtime support for dynamic loading features on ELF-based platforms. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_ELFNIX_PLATFORM_H +#define ORC_RT_ELFNIX_PLATFORM_H + +#include "common.h" +#include "executor_address.h" + +// Atexit functions. +ORC_RT_INTERFACE int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle); +ORC_RT_INTERFACE int __orc_rt_elfnix_atexit(void (*func)(void *)); +ORC_RT_INTERFACE void __orc_rt_elfnix_cxa_finalize(void *dso_handle); + +// dlfcn functions. +ORC_RT_INTERFACE const char *__orc_rt_elfnix_jit_dlerror(); +ORC_RT_INTERFACE void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode); +ORC_RT_INTERFACE int __orc_rt_elfnix_jit_dlclose(void *dso_handle); +ORC_RT_INTERFACE void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, + const char *symbol); + +namespace __orc_rt { +namespace elfnix { + +struct ELFNixPerObjectSectionsToRegister { + ExecutorAddrRange EHFrameSection; + ExecutorAddrRange ThreadDataSection; +}; + +struct ELFNixJITDylibInitializers { + using SectionList = std::vector<ExecutorAddrRange>; + + ELFNixJITDylibInitializers() = default; + ELFNixJITDylibInitializers(std::string Name, ExecutorAddr DSOHandleAddress) + : Name(std::move(Name)), DSOHandleAddress(std::move(DSOHandleAddress)) {} + + std::string Name; + ExecutorAddr DSOHandleAddress; + + std::vector<std::pair<std::string, SectionList>> InitSections; +}; + +class ELFNixJITDylibDeinitializers {}; + +using ELFNixJITDylibInitializerSequence = + std::vector<ELFNixJITDylibInitializers>; + +using ELFNixJITDylibDeinitializerSequence = + std::vector<ELFNixJITDylibDeinitializers>; + +enum dlopen_mode : int { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 +}; + +} // end namespace elfnix + +using SPSELFNixPerObjectSectionsToRegister = + SPSTuple<SPSExecutorAddrRange, SPSExecutorAddrRange>; + +template <> +class SPSSerializationTraits<SPSELFNixPerObjectSectionsToRegister, + elfnix::ELFNixPerObjectSectionsToRegister> { + +public: + static size_t size(const elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) { + return SPSELFNixPerObjectSectionsToRegister::AsArgList::size( + MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } + + static bool + serialize(SPSOutputBuffer &OB, + const elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) { + return SPSELFNixPerObjectSectionsToRegister::AsArgList::serialize( + OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } + + static bool deserialize(SPSInputBuffer &IB, + elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) { + return SPSELFNixPerObjectSectionsToRegister::AsArgList::deserialize( + IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } +}; + +using SPSNamedExecutorAddrRangeSequenceMap = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRangeSequence>>; + +using SPSELFNixJITDylibInitializers = + SPSTuple<SPSString, SPSExecutorAddr, SPSNamedExecutorAddrRangeSequenceMap>; + +using SPSELFNixJITDylibInitializerSequence = + SPSSequence<SPSELFNixJITDylibInitializers>; + +/// Serialization traits for ELFNixJITDylibInitializers. +template <> +class SPSSerializationTraits<SPSELFNixJITDylibInitializers, + elfnix::ELFNixJITDylibInitializers> { +public: + static size_t size(const elfnix::ELFNixJITDylibInitializers &MOJDIs) { + return SPSELFNixJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const elfnix::ELFNixJITDylibInitializers &MOJDIs) { + return SPSELFNixJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + elfnix::ELFNixJITDylibInitializers &MOJDIs) { + return SPSELFNixJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections); + } +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_ELFNIX_PLATFORM_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.aarch64.S b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.aarch64.S new file mode 100644 index 000000000000..8dcdd535be8a --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.aarch64.S @@ -0,0 +1,94 @@ +//===-- elfnix_tlv.aarch64.s ---------------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +// The content of this file is aarch64-only +#if defined(__arm64__) || defined(__aarch64__) + +#define REGISTER_SAVE_SPACE_SIZE 32 * 24 + + .text + + // returns address of TLV in x0, all other registers preserved + // TODO: add fast-path for repeat access + .globl ___orc_rt_elfnix_tlsdesc_resolver +___orc_rt_elfnix_tlsdesc_resolver: + sub sp, sp, #REGISTER_SAVE_SPACE_SIZE + stp x29, x30, [sp, #16 * 1] + stp x27, x28, [sp, #16 * 2] + stp x25, x26, [sp, #16 * 3] + stp x23, x24, [sp, #16 * 4] + stp x21, x22, [sp, #16 * 5] + stp x19, x20, [sp, #16 * 6] + stp x17, x18, [sp, #16 * 7] + stp x15, x16, [sp, #16 * 8] + stp x13, x14, [sp, #16 * 9] + stp x11, x12, [sp, #16 * 10] + stp x9, x10, [sp, #16 * 11] + stp x7, x8, [sp, #16 * 12] + stp x5, x6, [sp, #16 * 13] + stp x3, x4, [sp, #16 * 14] + stp x1, x2, [sp, #16 * 15] + stp q30, q31, [sp, #32 * 8] + stp q28, q29, [sp, #32 * 9] + stp q26, q27, [sp, #32 * 10] + stp q24, q25, [sp, #32 * 11] + stp q22, q23, [sp, #32 * 12] + stp q20, q21, [sp, #32 * 13] + stp q18, q19, [sp, #32 * 14] + stp q16, q17, [sp, #32 * 15] + stp q14, q15, [sp, #32 * 16] + stp q12, q13, [sp, #32 * 17] + stp q10, q11, [sp, #32 * 18] + stp q8, q9, [sp, #32 * 19] + stp q6, q7, [sp, #32 * 20] + stp q4, q5, [sp, #32 * 21] + stp q2, q3, [sp, #32 * 22] + stp q0, q1, [sp, #32 * 23] + + mrs x1, TPIDR_EL0 // get thread pointer + bl ___orc_rt_elfnix_tlsdesc_resolver_impl + + ldp q0, q1, [sp, #32 * 23] + ldp q2, q3, [sp, #32 * 22] + ldp q4, q5, [sp, #32 * 21] + ldp q6, q7, [sp, #32 * 20] + ldp q8, q9, [sp, #32 * 19] + ldp q10, q11, [sp, #32 * 18] + ldp q12, q13, [sp, #32 * 17] + ldp q14, q15, [sp, #32 * 16] + ldp q16, q17, [sp, #32 * 15] + ldp q18, q19, [sp, #32 * 14] + ldp q20, q21, [sp, #32 * 13] + ldp q22, q23, [sp, #32 * 12] + ldp q24, q25, [sp, #32 * 11] + ldp q26, q27, [sp, #32 * 10] + ldp q28, q29, [sp, #32 * 9] + ldp q30, q31, [sp, #32 * 8] + ldp x1, x2, [sp, #16 * 15] + ldp x3, x4, [sp, #16 * 14] + ldp x5, x6, [sp, #16 * 13] + ldp x7, x8, [sp, #16 * 12] + ldp x9, x10, [sp, #16 * 11] + ldp x11, x12, [sp, #16 * 10] + ldp x13, x14, [sp, #16 * 9] + ldp x15, x16, [sp, #16 * 8] + ldp x17, x18, [sp, #16 * 7] + ldp x19, x20, [sp, #16 * 6] + ldp x21, x22, [sp, #16 * 5] + ldp x23, x24, [sp, #16 * 4] + ldp x25, x26, [sp, #16 * 3] + ldp x27, x28, [sp, #16 * 2] + ldp x29, x30, [sp, #16 * 1] + add sp, sp, #REGISTER_SAVE_SPACE_SIZE + ret + +#endif // defined(__arm64__) || defined(__aarch64__) diff --git a/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.ppc64.S b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.ppc64.S new file mode 100644 index 000000000000..84854795dba1 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.ppc64.S @@ -0,0 +1,33 @@ +//===-- orc_rt_elfnix_tls.ppc64.s -------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +// The content of this file is PowerPC64 only. +#if defined(__powerpc64__) + + .text + // TODO: add fast-path for repeat access. + // See https://github.com/llvm/llvm-project/issues/51162. + .global ___orc_rt_elfnix_tls_get_addr +___orc_rt_elfnix_tls_get_addr: + addis 2, 12, .TOC.-___orc_rt_elfnix_tls_get_addr@ha + addi 2, 2, .TOC.-___orc_rt_elfnix_tls_get_addr@l + mflr 0 + std 0, 16(1) + stdu 1, -32(1) + bl __orc_rt_elfnix_tls_get_addr_impl + nop + addi 1, 1, 32 + ld 0, 16(1) + mtlr 0 + blr + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.x86-64.S b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.x86-64.S new file mode 100644 index 000000000000..b3e0bef00867 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/elfnix_tls.x86-64.S @@ -0,0 +1,64 @@ + +//===-- orc_rt_elfnix_tls_x86-64.s -------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +// The content of this file is x86_64-only +#if defined(__x86_64__) + +#define REGISTER_SAVE_SPACE_SIZE 512 + + .text + + // returns address of TLV in %rax, all other registers preserved + .globl ___orc_rt_elfnix_tls_get_addr +___orc_rt_elfnix_tls_get_addr: + pushq %rbp + movq %rsp, %rbp + subq $REGISTER_SAVE_SPACE_SIZE, %rsp + movq %rcx, -16(%rbp) + movq %rdx, -24(%rbp) + movq %rsi, -32(%rbp) + movq %rdi, -40(%rbp) + movq %r8, -48(%rbp) + movq %r9, -56(%rbp) + movq %r10, -64(%rbp) + movq %r11, -72(%rbp) + movdqa %xmm0, -128(%rbp) + movdqa %xmm1, -144(%rbp) + movdqa %xmm2, -160(%rbp) + movdqa %xmm3, -176(%rbp) + movdqa %xmm4, -192(%rbp) + movdqa %xmm5, -208(%rbp) + movdqa %xmm6, -224(%rbp) + movdqa %xmm7, -240(%rbp) + call __orc_rt_elfnix_tls_get_addr_impl + movq -16(%rbp), %rcx + movq -24(%rbp), %rdx + movq -32(%rbp), %rsi + movq -40(%rbp), %rdi + movq -48(%rbp), %r8 + movq -56(%rbp), %r9 + movq -64(%rbp), %r10 + movq -72(%rbp), %r11 + movdqa -128(%rbp), %xmm0 + movdqa -144(%rbp), %xmm1 + movdqa -160(%rbp), %xmm2 + movdqa -176(%rbp), %xmm3 + movdqa -192(%rbp), %xmm4 + movdqa -208(%rbp), %xmm5 + movdqa -224(%rbp), %xmm6 + movdqa -240(%rbp), %xmm7 + addq $REGISTER_SAVE_SPACE_SIZE, %rsp + popq %rbp + ret + +#endif // defined(__x86_64__) diff --git a/contrib/llvm-project/compiler-rt/lib/orc/endianness.h b/contrib/llvm-project/compiler-rt/lib/orc/endianness.h new file mode 100644 index 000000000000..4ee5505ce6dd --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/endianness.h @@ -0,0 +1,143 @@ +//===- endian.h - Endianness support ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares generic and optimized functions to swap the byte order of +// an integral type. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_ENDIAN_H +#define ORC_RT_ENDIAN_H + +#include <cstddef> +#include <cstdint> +#include <type_traits> +#if defined(_MSC_VER) && !defined(_DEBUG) +#include <stdlib.h> +#endif + +#if defined(__linux__) || defined(__GNU__) || defined(__HAIKU__) || \ + defined(__Fuchsia__) || defined(__EMSCRIPTEN__) +#include <endian.h> +#elif defined(_AIX) +#include <sys/machine.h> +#elif defined(__sun) +/* Solaris provides _BIG_ENDIAN/_LITTLE_ENDIAN selector in sys/types.h */ +#include <sys/types.h> +#define BIG_ENDIAN 4321 +#define LITTLE_ENDIAN 1234 +#if defined(_BIG_ENDIAN) +#define BYTE_ORDER BIG_ENDIAN +#else +#define BYTE_ORDER LITTLE_ENDIAN +#endif +#elif defined(__MVS__) +#define BIG_ENDIAN 4321 +#define LITTLE_ENDIAN 1234 +#define BYTE_ORDER BIG_ENDIAN +#else +#if !defined(BYTE_ORDER) && !defined(_WIN32) +#include <machine/endian.h> +#endif +#endif + +namespace __orc_rt { + +/// ByteSwap_16 - This function returns a byte-swapped representation of +/// the 16-bit argument. +inline uint16_t ByteSwap_16(uint16_t value) { +#if defined(_MSC_VER) && !defined(_DEBUG) + // The DLL version of the runtime lacks these functions (bug!?), but in a + // release build they're replaced with BSWAP instructions anyway. + return _byteswap_ushort(value); +#else + uint16_t Hi = value << 8; + uint16_t Lo = value >> 8; + return Hi | Lo; +#endif +} + +/// This function returns a byte-swapped representation of the 32-bit argument. +inline uint32_t ByteSwap_32(uint32_t value) { +#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC)) + return __builtin_bswap32(value); +#elif defined(_MSC_VER) && !defined(_DEBUG) + return _byteswap_ulong(value); +#else + uint32_t Byte0 = value & 0x000000FF; + uint32_t Byte1 = value & 0x0000FF00; + uint32_t Byte2 = value & 0x00FF0000; + uint32_t Byte3 = value & 0xFF000000; + return (Byte0 << 24) | (Byte1 << 8) | (Byte2 >> 8) | (Byte3 >> 24); +#endif +} + +/// This function returns a byte-swapped representation of the 64-bit argument. +inline uint64_t ByteSwap_64(uint64_t value) { +#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC)) + return __builtin_bswap64(value); +#elif defined(_MSC_VER) && !defined(_DEBUG) + return _byteswap_uint64(value); +#else + uint64_t Hi = ByteSwap_32(uint32_t(value)); + uint32_t Lo = ByteSwap_32(uint32_t(value >> 32)); + return (Hi << 32) | Lo; +#endif +} + +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN +constexpr bool IsBigEndianHost = true; +#else +constexpr bool IsBigEndianHost = false; +#endif + +static const bool IsLittleEndianHost = !IsBigEndianHost; + +inline unsigned char getSwappedBytes(unsigned char C) { return C; } +inline signed char getSwappedBytes(signed char C) { return C; } +inline char getSwappedBytes(char C) { return C; } + +inline unsigned short getSwappedBytes(unsigned short C) { + return ByteSwap_16(C); +} +inline signed short getSwappedBytes(signed short C) { return ByteSwap_16(C); } + +inline unsigned int getSwappedBytes(unsigned int C) { return ByteSwap_32(C); } +inline signed int getSwappedBytes(signed int C) { return ByteSwap_32(C); } + +inline unsigned long getSwappedBytes(unsigned long C) { + // Handle LLP64 and LP64 platforms. + return sizeof(long) == sizeof(int) ? ByteSwap_32((uint32_t)C) + : ByteSwap_64((uint64_t)C); +} +inline signed long getSwappedBytes(signed long C) { + // Handle LLP64 and LP64 platforms. + return sizeof(long) == sizeof(int) ? ByteSwap_32((uint32_t)C) + : ByteSwap_64((uint64_t)C); +} + +inline unsigned long long getSwappedBytes(unsigned long long C) { + return ByteSwap_64(C); +} +inline signed long long getSwappedBytes(signed long long C) { + return ByteSwap_64(C); +} + +template <typename T> +inline std::enable_if_t<std::is_enum<T>::value, T> getSwappedBytes(T C) { + return static_cast<T>( + getSwappedBytes(static_cast<std::underlying_type_t<T>>(C))); +} + +template <typename T> inline void swapByteOrder(T &Value) { + Value = getSwappedBytes(Value); +} + +} // end namespace __orc_rt + +#endif // ORC_RT_ENDIAN_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/error.h b/contrib/llvm-project/compiler-rt/lib/orc/error.h new file mode 100644 index 000000000000..b5da0769c637 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/error.h @@ -0,0 +1,426 @@ +//===-------- error.h - Enforced error checking for ORC RT ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_ERROR_H +#define ORC_RT_ERROR_H + +#include "compiler.h" +#include "extensible_rtti.h" +#include "stl_extras.h" + +#include <cassert> +#include <memory> +#include <string> +#include <type_traits> + +namespace __orc_rt { + +/// Base class for all errors. +class ErrorInfoBase : public RTTIExtends<ErrorInfoBase, RTTIRoot> { +public: + virtual std::string toString() const = 0; +}; + +/// Represents an environmental error. +class ORC_RT_NODISCARD Error { + + template <typename ErrT, typename... ArgTs> + friend Error make_error(ArgTs &&...Args); + + friend Error repackage_error(std::unique_ptr<ErrorInfoBase>); + + template <typename ErrT> friend std::unique_ptr<ErrT> error_cast(Error &); + + template <typename T> friend class Expected; + +public: + /// Destroy this error. Aborts if error was not checked, or was checked but + /// not handled. + ~Error() { assertIsChecked(); } + + Error(const Error &) = delete; + Error &operator=(const Error &) = delete; + + /// Move-construct an error. The newly constructed error is considered + /// unchecked, even if the source error had been checked. The original error + /// becomes a checked success value. + Error(Error &&Other) { + setChecked(true); + *this = std::move(Other); + } + + /// Move-assign an error value. The current error must represent success, you + /// you cannot overwrite an unhandled error. The current error is then + /// considered unchecked. The source error becomes a checked success value, + /// regardless of its original state. + Error &operator=(Error &&Other) { + // Don't allow overwriting of unchecked values. + assertIsChecked(); + setPtr(Other.getPtr()); + + // This Error is unchecked, even if the source error was checked. + setChecked(false); + + // Null out Other's payload and set its checked bit. + Other.setPtr(nullptr); + Other.setChecked(true); + + return *this; + } + + /// Create a success value. + static Error success() { return Error(); } + + /// Error values convert to true for failure values, false otherwise. + explicit operator bool() { + setChecked(getPtr() == nullptr); + return getPtr() != nullptr; + } + + /// Return true if this Error contains a failure value of the given type. + template <typename ErrT> bool isA() const { + return getPtr() && getPtr()->isA<ErrT>(); + } + +private: + Error() = default; + + Error(std::unique_ptr<ErrorInfoBase> ErrInfo) { + auto RawErrPtr = reinterpret_cast<uintptr_t>(ErrInfo.release()); + assert((RawErrPtr & 0x1) == 0 && "ErrorInfo is insufficiently aligned"); + ErrPtr = RawErrPtr | 0x1; + } + + void assertIsChecked() { + if (ORC_RT_UNLIKELY(!isChecked() || getPtr())) { + fprintf(stderr, "Error must be checked prior to destruction.\n"); + abort(); // Some sort of JIT program abort? + } + } + + template <typename ErrT = ErrorInfoBase> ErrT *getPtr() const { + return reinterpret_cast<ErrT *>(ErrPtr & ~uintptr_t(1)); + } + + void setPtr(ErrorInfoBase *Ptr) { + ErrPtr = (reinterpret_cast<uintptr_t>(Ptr) & ~uintptr_t(1)) | (ErrPtr & 1); + } + + bool isChecked() const { return ErrPtr & 0x1; } + + void setChecked(bool Checked) { ErrPtr = (ErrPtr & ~uintptr_t(1)) | Checked; } + + template <typename ErrT = ErrorInfoBase> std::unique_ptr<ErrT> takePayload() { + static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value, + "ErrT is not an ErrorInfoBase subclass"); + std::unique_ptr<ErrT> Tmp(getPtr<ErrT>()); + setPtr(nullptr); + setChecked(true); + return Tmp; + } + + uintptr_t ErrPtr = 0; +}; + +/// Construct an error of ErrT with the given arguments. +template <typename ErrT, typename... ArgTs> Error make_error(ArgTs &&...Args) { + static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value, + "ErrT is not an ErrorInfoBase subclass"); + return Error(std::make_unique<ErrT>(std::forward<ArgTs>(Args)...)); +} + +/// Construct an error of ErrT using a std::unique_ptr<ErrorInfoBase>. The +/// primary use-case for this is 're-packaging' errors after inspecting them +/// using error_cast, hence the name. +inline Error repackage_error(std::unique_ptr<ErrorInfoBase> EIB) { + return Error(std::move(EIB)); +} + +/// If the argument is an error of type ErrT then this function unpacks it +/// and returns a std::unique_ptr<ErrT>. Otherwise returns a nullptr and +/// leaves the error untouched. Common usage looks like: +/// +/// \code{.cpp} +/// if (Error E = foo()) { +/// if (auto EV1 = error_cast<ErrorType1>(E)) { +/// // use unwrapped EV1 value. +/// } else if (EV2 = error_cast<ErrorType2>(E)) { +/// // use unwrapped EV2 value. +/// } ... +/// } +/// \endcode +template <typename ErrT> std::unique_ptr<ErrT> error_cast(Error &Err) { + static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value, + "ErrT is not an ErrorInfoBase subclass"); + if (Err.isA<ErrT>()) + return Err.takePayload<ErrT>(); + return nullptr; +} + +/// Helper for Errors used as out-parameters. +/// Sets the 'checked' flag on construction, resets it on destruction. +class ErrorAsOutParameter { +public: + ErrorAsOutParameter(Error *Err) : Err(Err) { + // Raise the checked bit if Err is success. + if (Err) + (void)!!*Err; + } + + ~ErrorAsOutParameter() { + // Clear the checked bit. + if (Err && !*Err) + *Err = Error::success(); + } + +private: + Error *Err; +}; + +template <typename T> class ORC_RT_NODISCARD Expected { + + template <class OtherT> friend class Expected; + + static constexpr bool IsRef = std::is_reference<T>::value; + using wrap = std::reference_wrapper<std::remove_reference_t<T>>; + using error_type = std::unique_ptr<ErrorInfoBase>; + using storage_type = std::conditional_t<IsRef, wrap, T>; + using value_type = T; + + using reference = std::remove_reference_t<T> &; + using const_reference = const std::remove_reference_t<T> &; + using pointer = std::remove_reference_t<T> *; + using const_pointer = const std::remove_reference_t<T> *; + +public: + /// Create an Expected from a failure value. + Expected(Error Err) : HasError(true), Unchecked(true) { + assert(Err && "Cannot create Expected<T> from Error success value"); + new (getErrorStorage()) error_type(Err.takePayload()); + } + + /// Create an Expected from a T value. + template <typename OtherT> + Expected(OtherT &&Val, + std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) + : HasError(false), Unchecked(true) { + new (getStorage()) storage_type(std::forward<OtherT>(Val)); + } + + /// Move-construct an Expected<T> from an Expected<OtherT>. + Expected(Expected &&Other) { moveConstruct(std::move(Other)); } + + /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT + /// must be convertible to T. + template <class OtherT> + Expected( + Expected<OtherT> &&Other, + std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) { + moveConstruct(std::move(Other)); + } + + /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT + /// isn't convertible to T. + template <class OtherT> + explicit Expected( + Expected<OtherT> &&Other, + std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr) { + moveConstruct(std::move(Other)); + } + + /// Move-assign from another Expected<T>. + Expected &operator=(Expected &&Other) { + moveAssign(std::move(Other)); + return *this; + } + + /// Destroy an Expected<T>. + ~Expected() { + assertIsChecked(); + if (!HasError) + getStorage()->~storage_type(); + else + getErrorStorage()->~error_type(); + } + + /// Returns true if this Expected value is in a success state (holding a T), + /// and false if this Expected value is in a failure state. + explicit operator bool() { + Unchecked = HasError; + return !HasError; + } + + /// Returns true if this Expected value holds an Error of type error_type. + template <typename ErrT> bool isFailureOfType() const { + return HasError && (*getErrorStorage())->template isFailureOfType<ErrT>(); + } + + /// Take ownership of the stored error. + /// + /// If this Expected value is in a success state (holding a T) then this + /// method is a no-op and returns Error::success. + /// + /// If thsi Expected value is in a failure state (holding an Error) then this + /// method returns the contained error and leaves this Expected in an + /// 'empty' state from which it may be safely destructed but not otherwise + /// accessed. + Error takeError() { + Unchecked = false; + return HasError ? Error(std::move(*getErrorStorage())) : Error::success(); + } + + /// Returns a pointer to the stored T value. + pointer operator->() { + assertIsChecked(); + return toPointer(getStorage()); + } + + /// Returns a pointer to the stored T value. + const_pointer operator->() const { + assertIsChecked(); + return toPointer(getStorage()); + } + + /// Returns a reference to the stored T value. + reference operator*() { + assertIsChecked(); + return *getStorage(); + } + + /// Returns a reference to the stored T value. + const_reference operator*() const { + assertIsChecked(); + return *getStorage(); + } + +private: + template <class T1> + static bool compareThisIfSameType(const T1 &a, const T1 &b) { + return &a == &b; + } + + template <class T1, class T2> + static bool compareThisIfSameType(const T1 &a, const T2 &b) { + return false; + } + + template <class OtherT> void moveConstruct(Expected<OtherT> &&Other) { + HasError = Other.HasError; + Unchecked = true; + Other.Unchecked = false; + + if (!HasError) + new (getStorage()) storage_type(std::move(*Other.getStorage())); + else + new (getErrorStorage()) error_type(std::move(*Other.getErrorStorage())); + } + + template <class OtherT> void moveAssign(Expected<OtherT> &&Other) { + assertIsChecked(); + + if (compareThisIfSameType(*this, Other)) + return; + + this->~Expected(); + new (this) Expected(std::move(Other)); + } + + pointer toPointer(pointer Val) { return Val; } + + const_pointer toPointer(const_pointer Val) const { return Val; } + + pointer toPointer(wrap *Val) { return &Val->get(); } + + const_pointer toPointer(const wrap *Val) const { return &Val->get(); } + + storage_type *getStorage() { + assert(!HasError && "Cannot get value when an error exists!"); + return reinterpret_cast<storage_type *>(&TStorage); + } + + const storage_type *getStorage() const { + assert(!HasError && "Cannot get value when an error exists!"); + return reinterpret_cast<const storage_type *>(&TStorage); + } + + error_type *getErrorStorage() { + assert(HasError && "Cannot get error when a value exists!"); + return reinterpret_cast<error_type *>(&ErrorStorage); + } + + const error_type *getErrorStorage() const { + assert(HasError && "Cannot get error when a value exists!"); + return reinterpret_cast<const error_type *>(&ErrorStorage); + } + + void assertIsChecked() { + if (ORC_RT_UNLIKELY(Unchecked)) { + fprintf(stderr, + "Expected<T> must be checked before access or destruction.\n"); + abort(); + } + } + + union { + std::aligned_union_t<1, storage_type> TStorage; + std::aligned_union_t<1, error_type> ErrorStorage; + }; + + bool HasError : 1; + bool Unchecked : 1; +}; + +/// Consume an error without doing anything. +inline void consumeError(Error Err) { + if (Err) + (void)error_cast<ErrorInfoBase>(Err); +} + +/// Consumes success values. It is a programmatic error to call this function +/// on a failure value. +inline void cantFail(Error Err) { + assert(!Err && "cantFail called on failure value"); + consumeError(std::move(Err)); +} + +/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic +/// error to call this function on a failure value. +template <typename T> T cantFail(Expected<T> E) { + assert(E && "cantFail called on failure value"); + consumeError(E.takeError()); + return std::move(*E); +} + +/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic +/// error to call this function on a failure value. +template <typename T> T &cantFail(Expected<T &> E) { + assert(E && "cantFail called on failure value"); + consumeError(E.takeError()); + return *E; +} + +/// Convert the given error to a string. The error value is consumed in the +/// process. +inline std::string toString(Error Err) { + if (auto EIB = error_cast<ErrorInfoBase>(Err)) + return EIB->toString(); + return {}; +} + +class StringError : public RTTIExtends<StringError, ErrorInfoBase> { +public: + StringError(std::string ErrMsg) : ErrMsg(std::move(ErrMsg)) {} + std::string toString() const override { return ErrMsg; } + +private: + std::string ErrMsg; +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_ERROR_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/executor_address.h b/contrib/llvm-project/compiler-rt/lib/orc/executor_address.h new file mode 100644 index 000000000000..1542ee96bd92 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/executor_address.h @@ -0,0 +1,263 @@ +//===------ ExecutorAddress.h - Executing process address -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Represents an address in the executing program. +// +// This file was derived from +// llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_EXECUTOR_ADDRESS_H +#define ORC_RT_EXECUTOR_ADDRESS_H + +#include "adt.h" +#include "simple_packed_serialization.h" + +#include <cassert> +#include <type_traits> + +namespace __orc_rt { + +using ExecutorAddrDiff = uint64_t; + +/// Represents an address in the executor process. +class ExecutorAddr { +public: + /// A wrap/unwrap function that leaves pointers unmodified. + template <typename T> using rawPtr = __orc_rt::identity<T *>; + + /// Default wrap function to use on this host. + template <typename T> using defaultWrap = rawPtr<T>; + + /// Default unwrap function to use on this host. + template <typename T> using defaultUnwrap = rawPtr<T>; + + /// Merges a tag into the raw address value: + /// P' = P | (TagValue << TagOffset). + class Tag { + public: + constexpr Tag(uintptr_t TagValue, uintptr_t TagOffset) + : TagMask(TagValue << TagOffset) {} + + template <typename T> constexpr T *operator()(T *P) { + return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(P) | TagMask); + } + + private: + uintptr_t TagMask; + }; + + /// Strips a tag of the given length from the given offset within the pointer: + /// P' = P & ~(((1 << TagLen) -1) << TagOffset) + class Untag { + public: + constexpr Untag(uintptr_t TagLen, uintptr_t TagOffset) + : UntagMask(~(((uintptr_t(1) << TagLen) - 1) << TagOffset)) {} + + template <typename T> constexpr T *operator()(T *P) { + return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(P) & UntagMask); + } + + private: + uintptr_t UntagMask; + }; + + ExecutorAddr() = default; + explicit ExecutorAddr(uint64_t Addr) : Addr(Addr) {} + + /// Create an ExecutorAddr from the given pointer. + template <typename T, typename UnwrapFn = defaultUnwrap<T>> + static ExecutorAddr fromPtr(T *Ptr, UnwrapFn &&Unwrap = UnwrapFn()) { + return ExecutorAddr( + static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Unwrap(Ptr)))); + } + + /// Cast this ExecutorAddr to a pointer of the given type. + template <typename T, typename WrapFn = defaultWrap<std::remove_pointer_t<T>>> + std::enable_if_t<std::is_pointer<T>::value, T> + toPtr(WrapFn &&Wrap = WrapFn()) const { + uintptr_t IntPtr = static_cast<uintptr_t>(Addr); + assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t"); + return Wrap(reinterpret_cast<T>(IntPtr)); + } + + /// Cast this ExecutorAddr to a pointer of the given function type. + template <typename T, typename WrapFn = defaultWrap<T>> + std::enable_if_t<std::is_function<T>::value, T *> + toPtr(WrapFn &&Wrap = WrapFn()) const { + uintptr_t IntPtr = static_cast<uintptr_t>(Addr); + assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t"); + return Wrap(reinterpret_cast<T *>(IntPtr)); + } + + uint64_t getValue() const { return Addr; } + void setValue(uint64_t Addr) { this->Addr = Addr; } + bool isNull() const { return Addr == 0; } + + explicit operator bool() const { return Addr != 0; } + + friend bool operator==(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr == RHS.Addr; + } + + friend bool operator!=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr != RHS.Addr; + } + + friend bool operator<(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr < RHS.Addr; + } + + friend bool operator<=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr <= RHS.Addr; + } + + friend bool operator>(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr > RHS.Addr; + } + + friend bool operator>=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr >= RHS.Addr; + } + + ExecutorAddr &operator++() { + ++Addr; + return *this; + } + ExecutorAddr &operator--() { + --Addr; + return *this; + } + ExecutorAddr operator++(int) { return ExecutorAddr(Addr++); } + ExecutorAddr operator--(int) { return ExecutorAddr(Addr++); } + + ExecutorAddr &operator+=(const ExecutorAddrDiff Delta) { + Addr += Delta; + return *this; + } + + ExecutorAddr &operator-=(const ExecutorAddrDiff Delta) { + Addr -= Delta; + return *this; + } + +private: + uint64_t Addr = 0; +}; + +/// Subtracting two addresses yields an offset. +inline ExecutorAddrDiff operator-(const ExecutorAddr &LHS, + const ExecutorAddr &RHS) { + return ExecutorAddrDiff(LHS.getValue() - RHS.getValue()); +} + +/// Adding an offset and an address yields an address. +inline ExecutorAddr operator+(const ExecutorAddr &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddr(LHS.getValue() + RHS); +} + +/// Adding an address and an offset yields an address. +inline ExecutorAddr operator+(const ExecutorAddrDiff &LHS, + const ExecutorAddr &RHS) { + return ExecutorAddr(LHS + RHS.getValue()); +} + +/// Represents an address range in the exceutor process. +struct ExecutorAddrRange { + ExecutorAddrRange() = default; + ExecutorAddrRange(ExecutorAddr Start, ExecutorAddr End) + : Start(Start), End(End) {} + ExecutorAddrRange(ExecutorAddr Start, ExecutorAddrDiff Size) + : Start(Start), End(Start + Size) {} + + bool empty() const { return Start == End; } + ExecutorAddrDiff size() const { return End - Start; } + + friend bool operator==(const ExecutorAddrRange &LHS, + const ExecutorAddrRange &RHS) { + return LHS.Start == RHS.Start && LHS.End == RHS.End; + } + friend bool operator!=(const ExecutorAddrRange &LHS, + const ExecutorAddrRange &RHS) { + return !(LHS == RHS); + } + bool contains(ExecutorAddr Addr) const { return Start <= Addr && Addr < End; } + bool overlaps(const ExecutorAddrRange &Other) { + return !(Other.End <= Start || End <= Other.Start); + } + + template <typename T> span<T> toSpan() const { + assert(size() % sizeof(T) == 0 && + "AddressRange is not a multiple of sizeof(T)"); + return span<T>(Start.toPtr<T *>(), size() / sizeof(T)); + } + + ExecutorAddr Start; + ExecutorAddr End; +}; + +/// SPS serializatior for ExecutorAddr. +template <> class SPSSerializationTraits<SPSExecutorAddr, ExecutorAddr> { +public: + static size_t size(const ExecutorAddr &EA) { + return SPSArgList<uint64_t>::size(EA.getValue()); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddr &EA) { + return SPSArgList<uint64_t>::serialize(BOB, EA.getValue()); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddr &EA) { + uint64_t Tmp; + if (!SPSArgList<uint64_t>::deserialize(BIB, Tmp)) + return false; + EA = ExecutorAddr(Tmp); + return true; + } +}; + +using SPSExecutorAddrRange = SPSTuple<SPSExecutorAddr, SPSExecutorAddr>; + +/// Serialization traits for address ranges. +template <> +class SPSSerializationTraits<SPSExecutorAddrRange, ExecutorAddrRange> { +public: + static size_t size(const ExecutorAddrRange &Value) { + return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::size(Value.Start, + Value.End); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddrRange &Value) { + return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::serialize( + BOB, Value.Start, Value.End); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddrRange &Value) { + return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::deserialize( + BIB, Value.Start, Value.End); + } +}; + +using SPSExecutorAddrRangeSequence = SPSSequence<SPSExecutorAddrRange>; + +} // End namespace __orc_rt + +namespace std { + +// Make ExecutorAddr hashable. +template <> struct hash<__orc_rt::ExecutorAddr> { + size_t operator()(const __orc_rt::ExecutorAddr &A) const { + return hash<uint64_t>()(A.getValue()); + } +}; + +} // namespace std + +#endif // ORC_RT_EXECUTOR_ADDRESS_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/executor_symbol_def.h b/contrib/llvm-project/compiler-rt/lib/orc/executor_symbol_def.h new file mode 100644 index 000000000000..454cefe525cf --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/executor_symbol_def.h @@ -0,0 +1,151 @@ +//===--------- ExecutorSymbolDef.h - (Addr, Flags) pair ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Represents a defining location for a symbol in the executing program. +// +// This file was derived from +// llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_EXECUTOR_SYMBOL_DEF_H +#define ORC_RT_EXECUTOR_SYMBOL_DEF_H + +#include "bitmask_enum.h" +#include "executor_address.h" +#include "simple_packed_serialization.h" + +namespace __orc_rt { + +/// Flags for symbols in the JIT. +class JITSymbolFlags { +public: + using UnderlyingType = uint8_t; + using TargetFlagsType = uint8_t; + + /// These values must be kept in sync with \c JITSymbolFlags in the JIT. + enum FlagNames : UnderlyingType { + None = 0, + HasError = 1U << 0, + Weak = 1U << 1, + Common = 1U << 2, + Absolute = 1U << 3, + Exported = 1U << 4, + Callable = 1U << 5, + MaterializationSideEffectsOnly = 1U << 6, + ORC_RT_MARK_AS_BITMASK_ENUM( // LargestValue = + MaterializationSideEffectsOnly) + }; + + /// Default-construct a JITSymbolFlags instance. + JITSymbolFlags() = default; + + /// Construct a JITSymbolFlags instance from the given flags and target + /// flags. + JITSymbolFlags(FlagNames Flags, TargetFlagsType TargetFlags) + : TargetFlags(TargetFlags), Flags(Flags) {} + + bool operator==(const JITSymbolFlags &RHS) const { + return Flags == RHS.Flags && TargetFlags == RHS.TargetFlags; + } + + /// Get the underlying flags value as an integer. + UnderlyingType getRawFlagsValue() const { + return static_cast<UnderlyingType>(Flags); + } + + /// Return a reference to the target-specific flags. + TargetFlagsType &getTargetFlags() { return TargetFlags; } + + /// Return a reference to the target-specific flags. + const TargetFlagsType &getTargetFlags() const { return TargetFlags; } + +private: + TargetFlagsType TargetFlags = 0; + FlagNames Flags = None; +}; + +/// Represents a defining location for a JIT symbol. +class ExecutorSymbolDef { +public: + ExecutorSymbolDef() = default; + ExecutorSymbolDef(ExecutorAddr Addr, JITSymbolFlags Flags) + : Addr(Addr), Flags(Flags) {} + + const ExecutorAddr &getAddress() const { return Addr; } + + const JITSymbolFlags &getFlags() const { return Flags; } + + friend bool operator==(const ExecutorSymbolDef &LHS, + const ExecutorSymbolDef &RHS) { + return LHS.getAddress() == RHS.getAddress() && + LHS.getFlags() == RHS.getFlags(); + } + +private: + ExecutorAddr Addr; + JITSymbolFlags Flags; +}; + +using SPSJITSymbolFlags = + SPSTuple<JITSymbolFlags::UnderlyingType, JITSymbolFlags::TargetFlagsType>; + +/// SPS serializatior for JITSymbolFlags. +template <> class SPSSerializationTraits<SPSJITSymbolFlags, JITSymbolFlags> { + using FlagsArgList = SPSJITSymbolFlags::AsArgList; + +public: + static size_t size(const JITSymbolFlags &F) { + return FlagsArgList::size(F.getRawFlagsValue(), F.getTargetFlags()); + } + + static bool serialize(SPSOutputBuffer &BOB, const JITSymbolFlags &F) { + return FlagsArgList::serialize(BOB, F.getRawFlagsValue(), + F.getTargetFlags()); + } + + static bool deserialize(SPSInputBuffer &BIB, JITSymbolFlags &F) { + JITSymbolFlags::UnderlyingType RawFlags; + JITSymbolFlags::TargetFlagsType TargetFlags; + if (!FlagsArgList::deserialize(BIB, RawFlags, TargetFlags)) + return false; + F = JITSymbolFlags{static_cast<JITSymbolFlags::FlagNames>(RawFlags), + TargetFlags}; + return true; + } +}; + +using SPSExecutorSymbolDef = SPSTuple<SPSExecutorAddr, SPSJITSymbolFlags>; + +/// SPS serializatior for ExecutorSymbolDef. +template <> +class SPSSerializationTraits<SPSExecutorSymbolDef, ExecutorSymbolDef> { + using DefArgList = SPSExecutorSymbolDef::AsArgList; + +public: + static size_t size(const ExecutorSymbolDef &ESD) { + return DefArgList::size(ESD.getAddress(), ESD.getFlags()); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorSymbolDef &ESD) { + return DefArgList::serialize(BOB, ESD.getAddress(), ESD.getFlags()); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorSymbolDef &ESD) { + ExecutorAddr Addr; + JITSymbolFlags Flags; + if (!DefArgList::deserialize(BIB, Addr, Flags)) + return false; + ESD = ExecutorSymbolDef{Addr, Flags}; + return true; + } +}; + +} // End namespace __orc_rt + +#endif // ORC_RT_EXECUTOR_SYMBOL_DEF_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/extensible_rtti.cpp b/contrib/llvm-project/compiler-rt/lib/orc/extensible_rtti.cpp new file mode 100644 index 000000000000..c6951a449a3d --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/extensible_rtti.cpp @@ -0,0 +1,24 @@ +//===- extensible_rtti.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +// Note: +// This source file was adapted from lib/Support/ExtensibleRTTI.cpp, however +// the data structures are not shared and the code need not be kept in sync. +// +//===----------------------------------------------------------------------===// + +#include "extensible_rtti.h" + +namespace __orc_rt { + +char RTTIRoot::ID = 0; +void RTTIRoot::anchor() {} + +} // end namespace __orc_rt diff --git a/contrib/llvm-project/compiler-rt/lib/orc/extensible_rtti.h b/contrib/llvm-project/compiler-rt/lib/orc/extensible_rtti.h new file mode 100644 index 000000000000..72f68242e7c4 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/extensible_rtti.h @@ -0,0 +1,145 @@ +//===------ extensible_rtti.h - Extensible RTTI for ORC RT ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// \file +// +// Provides an extensible RTTI mechanism, that can be used regardless of whether +// the runtime is built with -frtti or not. This is predominantly used to +// support error handling. +// +// The RTTIRoot class defines methods for comparing type ids. Implementations +// of these methods can be injected into new classes using the RTTIExtends +// class template. +// +// E.g. +// +// @code{.cpp} +// class MyBaseClass : public RTTIExtends<MyBaseClass, RTTIRoot> { +// public: +// static char ID; +// virtual void foo() = 0; +// }; +// +// class MyDerivedClass1 : public RTTIExtends<MyDerivedClass1, MyBaseClass> { +// public: +// static char ID; +// void foo() override {} +// }; +// +// class MyDerivedClass2 : public RTTIExtends<MyDerivedClass2, MyBaseClass> { +// public: +// static char ID; +// void foo() override {} +// }; +// +// char MyBaseClass::ID = 0; +// char MyDerivedClass1::ID = 0; +// char MyDerivedClass2:: ID = 0; +// +// void fn() { +// std::unique_ptr<MyBaseClass> B = std::make_unique<MyDerivedClass1>(); +// outs() << isa<MyBaseClass>(B) << "\n"; // Outputs "1". +// outs() << isa<MyDerivedClass1>(B) << "\n"; // Outputs "1". +// outs() << isa<MyDerivedClass2>(B) << "\n"; // Outputs "0'. +// } +// +// @endcode +// +// Note: +// This header was adapted from llvm/Support/ExtensibleRTTI.h, however the +// data structures are not shared and the code need not be kept in sync. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_EXTENSIBLE_RTTI_H +#define ORC_RT_EXTENSIBLE_RTTI_H + +namespace __orc_rt { + +template <typename ThisT, typename ParentT> class RTTIExtends; + +/// Base class for the extensible RTTI hierarchy. +/// +/// This class defines virtual methods, dynamicClassID and isA, that enable +/// type comparisons. +class RTTIRoot { +public: + virtual ~RTTIRoot() = default; + + /// Returns the class ID for this type. + static const void *classID() { return &ID; } + + /// Returns the class ID for the dynamic type of this RTTIRoot instance. + virtual const void *dynamicClassID() const = 0; + + /// Returns true if this class's ID matches the given class ID. + virtual bool isA(const void *const ClassID) const { + return ClassID == classID(); + } + + /// Check whether this instance is a subclass of QueryT. + template <typename QueryT> bool isA() const { return isA(QueryT::classID()); } + + static bool classof(const RTTIRoot *R) { return R->isA<RTTIRoot>(); } + +private: + virtual void anchor(); + + static char ID; +}; + +/// Inheritance utility for extensible RTTI. +/// +/// Supports single inheritance only: A class can only have one +/// ExtensibleRTTI-parent (i.e. a parent for which the isa<> test will work), +/// though it can have many non-ExtensibleRTTI parents. +/// +/// RTTIExtents uses CRTP so the first template argument to RTTIExtends is the +/// newly introduced type, and the *second* argument is the parent class. +/// +/// class MyType : public RTTIExtends<MyType, RTTIRoot> { +/// public: +/// static char ID; +/// }; +/// +/// class MyDerivedType : public RTTIExtends<MyDerivedType, MyType> { +/// public: +/// static char ID; +/// }; +/// +template <typename ThisT, typename ParentT> class RTTIExtends : public ParentT { +public: + // Inherit constructors and isA methods from ParentT. + using ParentT::isA; + using ParentT::ParentT; + + static char ID; + + static const void *classID() { return &ThisT::ID; } + + const void *dynamicClassID() const override { return &ThisT::ID; } + + bool isA(const void *const ClassID) const override { + return ClassID == classID() || ParentT::isA(ClassID); + } + + static bool classof(const RTTIRoot *R) { return R->isA<ThisT>(); } +}; + +template <typename ThisT, typename ParentT> +char RTTIExtends<ThisT, ParentT>::ID = 0; + +/// Returns true if the given value is an instance of the template type +/// parameter. +template <typename To, typename From> bool isa(const From &Value) { + return To::classof(&Value); +} + +} // end namespace __orc_rt + +#endif // ORC_RT_EXTENSIBLE_RTTI_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/interval_map.h b/contrib/llvm-project/compiler-rt/lib/orc/interval_map.h new file mode 100644 index 000000000000..8c1609d72f57 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/interval_map.h @@ -0,0 +1,168 @@ +//===--------- interval_map.h - A sorted interval map -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements a coalescing interval map. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_INTERVAL_MAP_H +#define ORC_RT_INTERVAL_MAP_H + +#include "adt.h" +#include <cassert> +#include <map> + +namespace __orc_rt { + +enum class IntervalCoalescing { Enabled, Disabled }; + +/// Maps intervals to keys with optional coalescing. +/// +/// NOTE: The interface is kept mostly compatible with LLVM's IntervalMap +/// collection to make it easy to swap over in the future if we choose +/// to. +template <typename KeyT, typename ValT> class IntervalMapBase { +private: + using KeyPairT = std::pair<KeyT, KeyT>; + + struct Compare { + using is_transparent = std::true_type; + bool operator()(const KeyPairT &LHS, const KeyPairT &RHS) const { + return LHS < RHS; + } + bool operator()(const KeyPairT &LHS, const KeyT &RHS) const { + return LHS.first < RHS; + } + bool operator()(const KeyT &LHS, const KeyPairT &RHS) const { + return LHS < RHS.first; + } + }; + + using ImplMap = std::map<KeyPairT, ValT, Compare>; + +public: + using iterator = typename ImplMap::iterator; + using const_iterator = typename ImplMap::const_iterator; + using size_type = typename ImplMap::size_type; + + bool empty() const { return Impl.empty(); } + + void clear() { Impl.clear(); } + + iterator begin() { return Impl.begin(); } + iterator end() { return Impl.end(); } + + const_iterator begin() const { return Impl.begin(); } + const_iterator end() const { return Impl.end(); } + + iterator find(KeyT K) { + // Early out if the key is clearly outside the range. + if (empty() || K < begin()->first.first || + K >= std::prev(end())->first.second) + return end(); + + auto I = Impl.upper_bound(K); + assert(I != begin() && "Should have hit early out above"); + I = std::prev(I); + if (K < I->first.second) + return I; + return end(); + } + + const_iterator find(KeyT K) const { + return const_cast<IntervalMapBase<KeyT, ValT> *>(this)->find(K); + } + + ValT lookup(KeyT K, ValT NotFound = ValT()) const { + auto I = find(K); + if (I == end()) + return NotFound; + return I->second; + } + + // Erase [KS, KE), which must be entirely containing within one existing + // range in the map. Removal is allowed to split the range. + void erase(KeyT KS, KeyT KE) { + if (empty()) + return; + + auto J = Impl.upper_bound(KS); + + // Check previous range. Bail out if range to remove is entirely after + // it. + auto I = std::prev(J); + if (KS >= I->first.second) + return; + + // Assert that range is wholly contained. + assert(KE <= I->first.second); + + auto Tmp = std::move(*I); + Impl.erase(I); + + // Split-right -- introduce right-split range. + if (KE < Tmp.first.second) { + Impl.insert( + J, std::make_pair(std::make_pair(KE, Tmp.first.second), Tmp.second)); + J = std::prev(J); + } + + // Split-left -- introduce left-split range. + if (KS > Tmp.first.first) + Impl.insert( + J, std::make_pair(std::make_pair(Tmp.first.first, KS), Tmp.second)); + } + +protected: + ImplMap Impl; +}; + +template <typename KeyT, typename ValT, IntervalCoalescing Coalescing> +class IntervalMap; + +template <typename KeyT, typename ValT> +class IntervalMap<KeyT, ValT, IntervalCoalescing::Enabled> + : public IntervalMapBase<KeyT, ValT> { +public: + // Coalescing insert. Requires that ValTs be equality-comparable. + void insert(KeyT KS, KeyT KE, ValT V) { + auto J = this->Impl.upper_bound(KS); + + // Coalesce-right if possible. Either way, J points at our insertion + // point. + if (J != this->end() && KE == J->first.first && J->second == V) { + KE = J->first.second; + auto Tmp = J++; + this->Impl.erase(Tmp); + } + + // Coalesce-left if possible. + if (J != this->begin()) { + auto I = std::prev(J); + if (I->first.second == KS && I->second == V) { + KS = I->first.first; + this->Impl.erase(I); + } + } + this->Impl.insert(J, std::make_pair(std::make_pair(KS, KE), std::move(V))); + } +}; + +template <typename KeyT, typename ValT> +class IntervalMap<KeyT, ValT, IntervalCoalescing::Disabled> + : public IntervalMapBase<KeyT, ValT> { +public: + // Non-coalescing insert. Does not require ValT to be equality-comparable. + void insert(KeyT KS, KeyT KE, ValT V) { + this->Impl.insert(std::make_pair(std::make_pair(KS, KE), std::move(V))); + } +}; + +} // End namespace __orc_rt + +#endif // ORC_RT_INTERVAL_MAP_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/interval_set.h b/contrib/llvm-project/compiler-rt/lib/orc/interval_set.h new file mode 100644 index 000000000000..20f40f9c7d37 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/interval_set.h @@ -0,0 +1,87 @@ +//===--------- interval_set.h - A sorted interval set -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements a coalescing interval set. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_INTERVAL_SET_H +#define ORC_RT_INTERVAL_SET_H + +#include "interval_map.h" + +namespace __orc_rt { + +/// Implements a coalescing interval set. +/// +/// Adjacent intervals are coalesced. +/// +/// NOTE: The interface is kept mostly compatible with LLVM's IntervalMap +/// collection to make it easy to swap over in the future if we choose +/// to. +template <typename KeyT, IntervalCoalescing Coalescing> +class IntervalSet { +private: + using ImplMap = IntervalMap<KeyT, std::monostate, Coalescing>; +public: + + using value_type = std::pair<KeyT, KeyT>; + + class const_iterator { + friend class IntervalSet; + public: + using difference_type = typename ImplMap::iterator::difference_type; + using value_type = IntervalSet::value_type; + using pointer = const value_type *; + using reference = const value_type &; + using iterator_category = std::input_iterator_tag; + + const_iterator() = default; + const value_type &operator*() const { return I->first; } + const value_type *operator->() const { return &I->first; } + const_iterator &operator++() { ++I; return *this; } + const_iterator operator++(int) { auto Tmp = I; ++I; return Tmp; } + friend bool operator==(const const_iterator &LHS, + const const_iterator &RHS) { + return LHS.I == RHS.I; + } + friend bool operator!=(const const_iterator &LHS, + const const_iterator &RHS) { + return LHS.I != RHS.I; + } + private: + const_iterator(typename ImplMap::const_iterator I) : I(std::move(I)) {} + typename ImplMap::const_iterator I; + }; + + bool empty() const { return Map.empty(); } + + void clear() { Map.clear(); } + + const_iterator begin() const { return const_iterator(Map.begin()); } + const_iterator end() const { return const_iterator(Map.end()); } + + const_iterator find(KeyT K) const { + return const_iterator(Map.find(K)); + } + + void insert(KeyT KS, KeyT KE) { + Map.insert(std::move(KS), std::move(KE), std::monostate()); + } + + void erase(KeyT KS, KeyT KE) { + Map.erase(KS, KE); + } + +private: + ImplMap Map; +}; + +} // End namespace __orc_rt + +#endif // ORC_RT_INTERVAL_SET_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/log_error_to_stderr.cpp b/contrib/llvm-project/compiler-rt/lib/orc/log_error_to_stderr.cpp new file mode 100644 index 000000000000..4fabbc0d212a --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/log_error_to_stderr.cpp @@ -0,0 +1,19 @@ +//===-- log_error_to_stderr.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "compiler.h" + +#include <stdio.h> + +ORC_RT_INTERFACE void __orc_rt_log_error_to_stderr(const char *ErrMsg) { + fprintf(stderr, "orc runtime error: %s\n", ErrMsg); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/macho_platform.cpp b/contrib/llvm-project/compiler-rt/lib/orc/macho_platform.cpp new file mode 100644 index 000000000000..340846f5f900 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/macho_platform.cpp @@ -0,0 +1,1550 @@ +//===- macho_platform.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code required to load the rest of the MachO runtime. +// +//===----------------------------------------------------------------------===// + +#include "macho_platform.h" +#include "bitmask_enum.h" +#include "common.h" +#include "debug.h" +#include "error.h" +#include "interval_map.h" +#include "wrapper_function_utils.h" + +#include <algorithm> +#include <ios> +#include <map> +#include <mutex> +#include <sstream> +#include <string_view> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#define DEBUG_TYPE "macho_platform" + +using namespace __orc_rt; +using namespace __orc_rt::macho; + +// Declare function tags for functions in the JIT process. +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_symbols_tag) + +struct objc_image_info; +struct mach_header; + +// Objective-C registration functions. +// These are weakly imported. If the Objective-C runtime has not been loaded +// then code containing Objective-C sections will generate an error. +extern "C" void +_objc_map_images(unsigned count, const char *const paths[], + const mach_header *const mhdrs[]) ORC_RT_WEAK_IMPORT; + +extern "C" void _objc_load_image(const char *path, + const mach_header *mh) ORC_RT_WEAK_IMPORT; + +// Libunwind prototypes. +struct unw_dynamic_unwind_sections { + uintptr_t dso_base; + uintptr_t dwarf_section; + size_t dwarf_section_length; + uintptr_t compact_unwind_section; + size_t compact_unwind_section_length; +}; + +typedef int (*unw_find_dynamic_unwind_sections)( + uintptr_t addr, struct unw_dynamic_unwind_sections *info); + +extern "C" int __unw_add_find_dynamic_unwind_sections( + unw_find_dynamic_unwind_sections find_dynamic_unwind_sections) + ORC_RT_WEAK_IMPORT; + +extern "C" int __unw_remove_find_dynamic_unwind_sections( + unw_find_dynamic_unwind_sections find_dynamic_unwind_sections) + ORC_RT_WEAK_IMPORT; + +namespace { + +struct MachOJITDylibDepInfo { + bool Sealed = false; + std::vector<ExecutorAddr> DepHeaders; +}; + +using MachOJITDylibDepInfoMap = + std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>; + +} // anonymous namespace + +namespace __orc_rt { + +using SPSMachOObjectPlatformSectionsMap = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>; + +using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>; + +using SPSMachOJITDylibDepInfoMap = + SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>; + +template <> +class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> { +public: + static size_t size(const MachOJITDylibDepInfo &JDI) { + return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders); + } + + static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) { + return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed, + JDI.DepHeaders); + } + + static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) { + return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed, + JDI.DepHeaders); + } +}; + +struct UnwindSectionInfo { + std::vector<ExecutorAddrRange> CodeRanges; + ExecutorAddrRange DwarfSection; + ExecutorAddrRange CompactUnwindSection; +}; + +using SPSUnwindSectionInfo = + SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange, + SPSExecutorAddrRange>; + +template <> +class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> { +public: + static size_t size(const UnwindSectionInfo &USI) { + return SPSUnwindSectionInfo::AsArgList::size( + USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); + } + + static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) { + return SPSUnwindSectionInfo::AsArgList::serialize( + OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); + } + + static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) { + return SPSUnwindSectionInfo::AsArgList::deserialize( + IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); + } +}; + +} // namespace __orc_rt + +namespace { +struct TLVDescriptor { + void *(*Thunk)(TLVDescriptor *) = nullptr; + unsigned long Key = 0; + unsigned long DataAddress = 0; +}; + +class MachOPlatformRuntimeState { +public: + // Used internally by MachOPlatformRuntimeState, but made public to enable + // serialization. + enum class MachOExecutorSymbolFlags : uint8_t { + None = 0, + Weak = 1U << 0, + Callable = 1U << 1, + ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable) + }; + +private: + struct AtExitEntry { + void (*Func)(void *); + void *Arg; + }; + + using AtExitsVector = std::vector<AtExitEntry>; + + /// Used to manage sections of fixed-sized metadata records (e.g. pointer + /// sections, selector refs, etc.) + template <typename RecordElement> class RecordSectionsTracker { + public: + /// Add a section to the "new" list. + void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); } + + /// Returns true if there are new sections to process. + bool hasNewSections() const { return !New.empty(); } + + /// Returns the number of new sections to process. + size_t numNewSections() const { return New.size(); } + + /// Process all new sections. + template <typename ProcessSectionFunc> + std::enable_if_t<std::is_void_v< + std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>> + processNewSections(ProcessSectionFunc &&ProcessSection) { + for (auto &Sec : New) + ProcessSection(Sec); + moveNewToProcessed(); + } + + /// Proces all new sections with a fallible handler. + /// + /// Successfully handled sections will be moved to the Processed + /// list. + template <typename ProcessSectionFunc> + std::enable_if_t< + std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc, + span<RecordElement>>>, + Error> + processNewSections(ProcessSectionFunc &&ProcessSection) { + for (size_t I = 0; I != New.size(); ++I) { + if (auto Err = ProcessSection(New[I])) { + for (size_t J = 0; J != I; ++J) + Processed.push_back(New[J]); + New.erase(New.begin(), New.begin() + I); + return Err; + } + } + moveNewToProcessed(); + return Error::success(); + } + + /// Move all sections back to New for reprocessing. + void reset() { + moveNewToProcessed(); + New = std::move(Processed); + } + + /// Remove the section with the given range. + bool removeIfPresent(ExecutorAddrRange R) { + if (removeIfPresent(New, R)) + return true; + return removeIfPresent(Processed, R); + } + + private: + void moveNewToProcessed() { + if (Processed.empty()) + Processed = std::move(New); + else { + Processed.reserve(Processed.size() + New.size()); + std::copy(New.begin(), New.end(), std::back_inserter(Processed)); + New.clear(); + } + } + + bool removeIfPresent(std::vector<span<RecordElement>> &V, + ExecutorAddrRange R) { + auto RI = std::find_if( + V.rbegin(), V.rend(), + [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) { + return E.data() == RS.data(); + }); + if (RI != V.rend()) { + V.erase(std::next(RI).base()); + return true; + } + return false; + } + + std::vector<span<RecordElement>> Processed; + std::vector<span<RecordElement>> New; + }; + + struct UnwindSections { + UnwindSections(const UnwindSectionInfo &USI) + : DwarfSection(USI.DwarfSection.toSpan<char>()), + CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {} + + span<char> DwarfSection; + span<char> CompactUnwindSection; + }; + + using UnwindSectionsMap = + IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>; + + struct JITDylibState { + + using SymbolTableMap = + std::unordered_map<std::string_view, + std::pair<ExecutorAddr, MachOExecutorSymbolFlags>>; + + std::string Name; + void *Header = nullptr; + bool Sealed = false; + size_t LinkedAgainstRefCount = 0; + size_t DlRefCount = 0; + SymbolTableMap SymbolTable; + std::vector<JITDylibState *> Deps; + AtExitsVector AtExits; + const objc_image_info *ObjCImageInfo = nullptr; + std::unordered_map<void *, std::vector<char>> DataSectionContent; + std::unordered_map<void *, size_t> ZeroInitRanges; + UnwindSectionsMap UnwindSections; + RecordSectionsTracker<void (*)()> ModInitsSections; + RecordSectionsTracker<char> ObjCRuntimeRegistrationObjects; + + bool referenced() const { + return LinkedAgainstRefCount != 0 || DlRefCount != 0; + } + }; + +public: + static Error create(); + static MachOPlatformRuntimeState &get(); + static Error destroy(); + + MachOPlatformRuntimeState() = default; + + // Delete copy and move constructors. + MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete; + MachOPlatformRuntimeState & + operator=(const MachOPlatformRuntimeState &) = delete; + MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; + MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; + + Error initialize(); + Error shutdown(); + + Error registerJITDylib(std::string Name, void *Header); + Error deregisterJITDylib(void *Header); + Error registerThreadDataSection(span<const char> ThreadDataSection); + Error deregisterThreadDataSection(span<const char> ThreadDataSection); + Error registerObjectSymbolTable( + ExecutorAddr HeaderAddr, + const std::vector<std::tuple<ExecutorAddr, ExecutorAddr, + MachOExecutorSymbolFlags>> &Entries); + Error deregisterObjectSymbolTable( + ExecutorAddr HeaderAddr, + const std::vector<std::tuple<ExecutorAddr, ExecutorAddr, + MachOExecutorSymbolFlags>> &Entries); + Error registerObjectPlatformSections( + ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs); + Error deregisterObjectPlatformSections( + ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs); + + const char *dlerror(); + void *dlopen(std::string_view Name, int Mode); + int dlclose(void *DSOHandle); + void *dlsym(void *DSOHandle, const char *Symbol); + + int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); + void runAtExits(std::unique_lock<std::mutex> &JDStatesLock, + JITDylibState &JDS); + void runAtExits(void *DSOHandle); + + /// Returns the base address of the section containing ThreadData. + Expected<std::pair<const char *, size_t>> + getThreadDataSectionFor(const char *ThreadData); + +private: + JITDylibState *getJITDylibStateByHeader(void *DSOHandle); + JITDylibState *getJITDylibStateByName(std::string_view Path); + + /// Requests materialization of the given symbols. For each pair, the bool + /// element indicates whether the symbol is required (true) or weakly + /// referenced (false). + Error requestPushSymbols(JITDylibState &JDS, + span<std::pair<std::string_view, bool>> Symbols); + + /// Attempts to look up the given symbols locally, requesting a push from the + /// remote if they're not found. Results are written to the Result span, which + /// must have the same size as the Symbols span. + Error + lookupSymbols(JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock, + span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result, + span<std::pair<std::string_view, bool>> Symbols); + + bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info); + + static int findDynamicUnwindSections(uintptr_t addr, + unw_dynamic_unwind_sections *info); + static Error registerEHFrames(span<const char> EHFrameSection); + static Error deregisterEHFrames(span<const char> EHFrameSection); + + static Error registerObjCRegistrationObjects(JITDylibState &JDS); + static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock, + JITDylibState &JDS); + + Expected<void *> dlopenImpl(std::string_view Path, int Mode); + Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock, + JITDylibState &JDS); + Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock, + JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo); + + Error dlcloseImpl(void *DSOHandle); + Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock, + JITDylibState &JDS); + + static MachOPlatformRuntimeState *MOPS; + + bool UseCallbackStyleUnwindInfo = false; + + // FIXME: Move to thread-state. + std::string DLFcnError; + + // APIMutex guards against concurrent entry into key "dyld" API functions + // (e.g. dlopen, dlclose). + std::recursive_mutex DyldAPIMutex; + + // JDStatesMutex guards the data structures that hold JITDylib state. + std::mutex JDStatesMutex; + std::unordered_map<void *, JITDylibState> JDStates; + std::unordered_map<std::string_view, void *> JDNameToHeader; + + // ThreadDataSectionsMutex guards thread local data section state. + std::mutex ThreadDataSectionsMutex; + std::map<const char *, size_t> ThreadDataSections; +}; + +} // anonymous namespace + +namespace __orc_rt { + +class SPSMachOExecutorSymbolFlags; + +template <> +class SPSSerializationTraits< + SPSMachOExecutorSymbolFlags, + MachOPlatformRuntimeState::MachOExecutorSymbolFlags> { +private: + using UT = std::underlying_type_t< + MachOPlatformRuntimeState::MachOExecutorSymbolFlags>; + +public: + static size_t + size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) { + return sizeof(UT); + } + + static bool + serialize(SPSOutputBuffer &OB, + const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) { + return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF)); + } + + static bool + deserialize(SPSInputBuffer &IB, + MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) { + UT Tmp; + if (!SPSArgList<UT>::deserialize(IB, Tmp)) + return false; + SF = static_cast<MachOPlatformRuntimeState::MachOExecutorSymbolFlags>(Tmp); + return true; + } +}; + +} // namespace __orc_rt + +namespace { + +MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr; + +Error MachOPlatformRuntimeState::create() { + assert(!MOPS && "MachOPlatformRuntimeState should be null"); + MOPS = new MachOPlatformRuntimeState(); + return MOPS->initialize(); +} + +MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() { + assert(MOPS && "MachOPlatformRuntimeState not initialized"); + return *MOPS; +} + +Error MachOPlatformRuntimeState::destroy() { + assert(MOPS && "MachOPlatformRuntimeState not initialized"); + auto Err = MOPS->shutdown(); + delete MOPS; + return Err; +} + +Error MachOPlatformRuntimeState::initialize() { + UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections && + __unw_remove_find_dynamic_unwind_sections; + if (UseCallbackStyleUnwindInfo) { + ORC_RT_DEBUG({ + printdbg("__unw_add/remove_find_dynamic_unwind_sections available." + " Using callback-based frame info lookup.\n"); + }); + if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections)) + return make_error<StringError>( + "Could not register findDynamicUnwindSections"); + } else { + ORC_RT_DEBUG({ + printdbg("__unw_add/remove_find_dynamic_unwind_sections not available." + " Using classic frame info registration.\n"); + }); + } + return Error::success(); +} + +Error MachOPlatformRuntimeState::shutdown() { + if (UseCallbackStyleUnwindInfo) { + if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) { + ORC_RT_DEBUG( + { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); }); + } + } + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerJITDylib(std::string Name, + void *Header) { + ORC_RT_DEBUG({ + printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header); + }); + std::lock_guard<std::mutex> Lock(JDStatesMutex); + if (JDStates.count(Header)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (name = " << Name << ")"; + return make_error<StringError>(ErrStream.str()); + } + if (JDNameToHeader.count(Name)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (header = " << Header << ")"; + return make_error<StringError>(ErrStream.str()); + } + + auto &JDS = JDStates[Header]; + JDS.Name = std::move(Name); + JDS.Header = Header; + JDNameToHeader[JDS.Name] = Header; + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) { + std::lock_guard<std::mutex> Lock(JDStatesMutex); + auto I = JDStates.find(Header); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Attempted to deregister unrecognized header " << Header; + return make_error<StringError>(ErrStream.str()); + } + + // Remove std::string construction once we can use C++20. + auto J = JDNameToHeader.find( + std::string(I->second.Name.data(), I->second.Name.size())); + assert(J != JDNameToHeader.end() && + "Missing JDNameToHeader entry for JITDylib"); + + ORC_RT_DEBUG({ + printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(), + Header); + }); + + JDNameToHeader.erase(J); + JDStates.erase(I); + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerThreadDataSection( + span<const char> ThreadDataSection) { + std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); + if (I != ThreadDataSections.begin()) { + auto J = std::prev(I); + if (J->first + J->second > ThreadDataSection.data()) + return make_error<StringError>("Overlapping __thread_data sections"); + } + ThreadDataSections.insert( + I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterThreadDataSection( + span<const char> ThreadDataSection) { + std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.find(ThreadDataSection.data()); + if (I == ThreadDataSections.end()) + return make_error<StringError>("Attempt to deregister unknown thread data " + "section"); + ThreadDataSections.erase(I); + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerObjectSymbolTable( + ExecutorAddr HeaderAddr, + const std::vector<std::tuple<ExecutorAddr, ExecutorAddr, + MachOExecutorSymbolFlags>> &Entries) { + + std::lock_guard<std::mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "Could not register object platform sections for " + "unrecognized header " + << HeaderAddr.toPtr<void *>(); + return make_error<StringError>(ErrStream.str()); + } + + for (auto &[NameAddr, SymAddr, Flags] : Entries) + JDS->SymbolTable[NameAddr.toPtr<const char *>()] = {SymAddr, Flags}; + + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterObjectSymbolTable( + ExecutorAddr HeaderAddr, + const std::vector<std::tuple<ExecutorAddr, ExecutorAddr, + MachOExecutorSymbolFlags>> &Entries) { + + std::lock_guard<std::mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "Could not register object platform sections for " + "unrecognized header " + << HeaderAddr.toPtr<void *>(); + return make_error<StringError>(ErrStream.str()); + } + + for (auto &[NameAddr, SymAddr, Flags] : Entries) + JDS->SymbolTable.erase(NameAddr.toPtr<const char *>()); + + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerObjectPlatformSections( + ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) { + + // FIXME: Reject platform section registration after the JITDylib is + // sealed? + + ORC_RT_DEBUG({ + printdbg("MachOPlatform: Registering object sections for %p.\n", + HeaderAddr.toPtr<void *>()); + }); + + std::lock_guard<std::mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "Could not register object platform sections for " + "unrecognized header " + << HeaderAddr.toPtr<void *>(); + return make_error<StringError>(ErrStream.str()); + } + + if (UnwindInfo && UseCallbackStyleUnwindInfo) { + ORC_RT_DEBUG({ + printdbg(" Registering new-style unwind info for:\n" + " DWARF: %p -- %p\n" + " Compact-unwind: %p -- %p\n" + " for:\n", + UnwindInfo->DwarfSection.Start.toPtr<void *>(), + UnwindInfo->DwarfSection.End.toPtr<void *>(), + UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(), + UnwindInfo->CompactUnwindSection.End.toPtr<void *>()); + }); + for (auto &CodeRange : UnwindInfo->CodeRanges) { + JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(), + CodeRange.End.toPtr<char *>(), *UnwindInfo); + ORC_RT_DEBUG({ + printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(), + CodeRange.End.toPtr<void *>()); + }); + } + } + + for (auto &KV : Secs) { + // FIXME: Validate section ranges? + if (KV.first == "__TEXT,__eh_frame") { + if (!UseCallbackStyleUnwindInfo) { + // Use classic libunwind registration. + if (auto Err = registerEHFrames(KV.second.toSpan<const char>())) + return Err; + } + } else if (KV.first == "__DATA,__data") { + assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) && + "Address already registered."); + auto S = KV.second.toSpan<char>(); + JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] = + std::vector<char>(S.begin(), S.end()); + } else if (KV.first == "__DATA,__common") { + JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size(); + } else if (KV.first == "__DATA,__thread_data") { + if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>())) + return Err; + } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject") + JDS->ObjCRuntimeRegistrationObjects.add(KV.second.toSpan<char>()); + else if (KV.first == "__DATA,__mod_init_func") + JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>()); + else { + // Should this be a warning instead? + return make_error<StringError>( + "Encountered unexpected section " + + std::string(KV.first.data(), KV.first.size()) + + " while registering object platform sections"); + } + } + + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterObjectPlatformSections( + ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) { + // TODO: Make this more efficient? (maybe unnecessary if removal is rare?) + // TODO: Add a JITDylib prepare-for-teardown operation that clears all + // registered sections, causing this function to take the fast-path. + ORC_RT_DEBUG({ + printdbg("MachOPlatform: Deregistering object sections for %p.\n", + HeaderAddr.toPtr<void *>()); + }); + + std::lock_guard<std::mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "Could not register object platform sections for unrecognized " + "header " + << HeaderAddr.toPtr<void *>(); + return make_error<StringError>(ErrStream.str()); + } + + // FIXME: Implement faster-path by returning immediately if JDS is being + // torn down entirely? + + // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains + // any Swift or ObjC. Once this happens we can clear (and no longer record) + // data section content, as the library could never be re-initialized. + + if (UnwindInfo && UseCallbackStyleUnwindInfo) { + ORC_RT_DEBUG({ + printdbg(" Deregistering new-style unwind info for:\n" + " DWARF: %p -- %p\n" + " Compact-unwind: %p -- %p\n" + " for:\n", + UnwindInfo->DwarfSection.Start.toPtr<void *>(), + UnwindInfo->DwarfSection.End.toPtr<void *>(), + UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(), + UnwindInfo->CompactUnwindSection.End.toPtr<void *>()); + }); + for (auto &CodeRange : UnwindInfo->CodeRanges) { + JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(), + CodeRange.End.toPtr<char *>()); + ORC_RT_DEBUG({ + printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(), + CodeRange.End.toPtr<void *>()); + }); + } + } + + for (auto &KV : Secs) { + // FIXME: Validate section ranges? + if (KV.first == "__TEXT,__eh_frame") { + if (!UseCallbackStyleUnwindInfo) { + // Use classic libunwind registration. + if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>())) + return Err; + } + } else if (KV.first == "__DATA,__data") { + JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>()); + } else if (KV.first == "__DATA,__common") { + JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>()); + } else if (KV.first == "__DATA,__thread_data") { + if (auto Err = + deregisterThreadDataSection(KV.second.toSpan<const char>())) + return Err; + } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject") + JDS->ObjCRuntimeRegistrationObjects.removeIfPresent(KV.second); + else if (KV.first == "__DATA,__mod_init_func") + JDS->ModInitsSections.removeIfPresent(KV.second); + else { + // Should this be a warning instead? + return make_error<StringError>( + "Encountered unexpected section " + + std::string(KV.first.data(), KV.first.size()) + + " while deregistering object platform sections"); + } + } + return Error::success(); +} + +const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } + +void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { + ORC_RT_DEBUG({ + std::string S(Path.data(), Path.size()); + printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str()); + }); + std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex); + if (auto H = dlopenImpl(Path, Mode)) + return *H; + else { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(H.takeError()); + return nullptr; + } +} + +int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { + ORC_RT_DEBUG({ + auto *JDS = getJITDylibStateByHeader(DSOHandle); + std::string DylibName; + if (JDS) { + std::string S; + printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str()); + } else + printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, + "invalid handle"); + }); + std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex); + if (auto Err = dlcloseImpl(DSOHandle)) { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(std::move(Err)); + return -1; + } + return 0; +} + +void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) { + std::unique_lock<std::mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(DSOHandle); + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "In call to dlsym, unrecognized header address " << DSOHandle; + DLFcnError = ErrStream.str(); + return nullptr; + } + + std::string MangledName = std::string("_") + Symbol; + std::pair<std::string_view, bool> Lookup(MangledName, false); + std::pair<ExecutorAddr, MachOExecutorSymbolFlags> Result; + + if (auto Err = lookupSymbols(*JDS, Lock, {&Result, 1}, {&Lookup, 1})) { + DLFcnError = toString(std::move(Err)); + return nullptr; + } + + // Sign callable symbols as functions, to match dyld. + if ((Result.second & MachOExecutorSymbolFlags::Callable) == + MachOExecutorSymbolFlags::Callable) + return reinterpret_cast<void *>(Result.first.toPtr<void(void)>()); + return Result.first.toPtr<void *>(); +} + +int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, + void *DSOHandle) { + // FIXME: Handle out-of-memory errors, returning -1 if OOM. + std::lock_guard<std::mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(DSOHandle); + if (!JDS) { + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::registerAtExit called with " + "unrecognized dso handle %p\n", + DSOHandle); + }); + return -1; + } + JDS->AtExits.push_back({F, Arg}); + return 0; +} + +void MachOPlatformRuntimeState::runAtExits( + std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { + auto AtExits = std::move(JDS.AtExits); + + // Unlock while running atexits, as they may trigger operations that modify + // JDStates. + JDStatesLock.unlock(); + while (!AtExits.empty()) { + auto &AE = AtExits.back(); + AE.Func(AE.Arg); + AtExits.pop_back(); + } + JDStatesLock.lock(); +} + +void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) { + std::unique_lock<std::mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(DSOHandle); + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized " + "dso_handle %p\n", + DSOHandle); + }); + if (JDS) + runAtExits(Lock, *JDS); +} + +Expected<std::pair<const char *, size_t>> +MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { + std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadData); + // Check that we have a valid entry covering this address. + if (I == ThreadDataSections.begin()) + return make_error<StringError>("No thread local data section for key"); + I = std::prev(I); + if (ThreadData >= I->first + I->second) + return make_error<StringError>("No thread local data section for key"); + return *I; +} + +MachOPlatformRuntimeState::JITDylibState * +MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) { + auto I = JDStates.find(DSOHandle); + if (I == JDStates.end()) { + I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first; + I->second.Header = DSOHandle; + } + return &I->second; +} + +MachOPlatformRuntimeState::JITDylibState * +MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { + // FIXME: Avoid creating string once we have C++20. + auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); + if (I != JDNameToHeader.end()) + return getJITDylibStateByHeader(I->second); + return nullptr; +} + +Error MachOPlatformRuntimeState::requestPushSymbols( + JITDylibState &JDS, span<std::pair<std::string_view, bool>> Symbols) { + Error OpErr = Error::success(); + if (auto Err = WrapperFunction<SPSError( + SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>)>:: + call(&__orc_rt_macho_push_symbols_tag, OpErr, + ExecutorAddr::fromPtr(JDS.Header), Symbols)) { + cantFail(std::move(OpErr)); + return std::move(Err); + } + return OpErr; +} + +Error MachOPlatformRuntimeState::lookupSymbols( + JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock, + span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result, + span<std::pair<std::string_view, bool>> Symbols) { + assert(JDStatesLock.owns_lock() && + "JDStatesLock should be locked at call-site"); + assert(Result.size() == Symbols.size() && + "Results and Symbols span sizes should match"); + + // Make an initial pass over the local symbol table. + std::vector<size_t> MissingSymbolIndexes; + for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) { + auto I = JDS.SymbolTable.find(Symbols[Idx].first); + if (I != JDS.SymbolTable.end()) + Result[Idx] = I->second; + else + MissingSymbolIndexes.push_back(Idx); + } + + // If everything has been resolved already then bail out early. + if (MissingSymbolIndexes.empty()) + return Error::success(); + + // Otherwise call back to the controller to try to request that the symbol + // be materialized. + std::vector<std::pair<std::string_view, bool>> MissingSymbols; + MissingSymbols.reserve(MissingSymbolIndexes.size()); + ORC_RT_DEBUG({ + printdbg("requesting push of %i missing symbols...\n", + MissingSymbolIndexes.size()); + }); + for (auto MissingIdx : MissingSymbolIndexes) + MissingSymbols.push_back(Symbols[MissingIdx]); + + JDStatesLock.unlock(); + if (auto Err = requestPushSymbols( + JDS, {MissingSymbols.data(), MissingSymbols.size()})) + return Err; + JDStatesLock.lock(); + + // Try to resolve the previously missing symbols locally. + std::vector<size_t> MissingRequiredSymbols; + for (auto MissingIdx : MissingSymbolIndexes) { + auto I = JDS.SymbolTable.find(Symbols[MissingIdx].first); + if (I != JDS.SymbolTable.end()) + Result[MissingIdx] = I->second; + else { + if (Symbols[MissingIdx].second) + MissingRequiredSymbols.push_back(MissingIdx); + else + Result[MissingIdx] = {ExecutorAddr(), {}}; + } + } + + // Error out if any missing symbols could not be resolved. + if (!MissingRequiredSymbols.empty()) { + std::ostringstream ErrStream; + ErrStream << "Lookup could not find required symbols: [ "; + for (auto MissingIdx : MissingRequiredSymbols) + ErrStream << "\"" << Symbols[MissingIdx].first << "\" "; + ErrStream << "]"; + return make_error<StringError>(ErrStream.str()); + } + + return Error::success(); +} + +// eh-frame registration functions. +// We expect these to be available for all processes. +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +template <typename HandleFDEFn> +void walkEHFrameSection(span<const char> EHFrameSection, + HandleFDEFn HandleFDE) { + const char *CurCFIRecord = EHFrameSection.data(); + uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + + while (CurCFIRecord != EHFrameSection.end() && Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); + + if (Offset != 0) + HandleFDE(CurCFIRecord); + + CurCFIRecord += Size; + Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + } +} + +bool MachOPlatformRuntimeState::lookupUnwindSections( + void *Addr, unw_dynamic_unwind_sections &Info) { + ORC_RT_DEBUG( + { printdbg("Tried to lookup unwind-info via new lookup call.\n"); }); + std::lock_guard<std::mutex> Lock(JDStatesMutex); + for (auto &KV : JDStates) { + auto &JD = KV.second; + auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr)); + if (I != JD.UnwindSections.end()) { + Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header); + Info.dwarf_section = + reinterpret_cast<uintptr_t>(I->second.DwarfSection.data()); + Info.dwarf_section_length = I->second.DwarfSection.size(); + Info.compact_unwind_section = + reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data()); + Info.compact_unwind_section_length = + I->second.CompactUnwindSection.size(); + return true; + } + } + return false; +} + +int MachOPlatformRuntimeState::findDynamicUnwindSections( + uintptr_t addr, unw_dynamic_unwind_sections *info) { + if (!info) + return 0; + return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr, + *info); +} + +Error MachOPlatformRuntimeState::registerEHFrames( + span<const char> EHFrameSection) { + walkEHFrameSection(EHFrameSection, __register_frame); + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterEHFrames( + span<const char> EHFrameSection) { + walkEHFrameSection(EHFrameSection, __deregister_frame); + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerObjCRegistrationObjects( + JITDylibState &JDS) { + ORC_RT_DEBUG(printdbg("Registering Objective-C / Swift metadata.\n")); + + std::vector<char *> RegObjBases; + JDS.ObjCRuntimeRegistrationObjects.processNewSections( + [&](span<char> RegObj) { RegObjBases.push_back(RegObj.data()); }); + + if (RegObjBases.empty()) + return Error::success(); + + if (!_objc_map_images || !_objc_load_image) + return make_error<StringError>( + "Could not register Objective-C / Swift metadata: _objc_map_images / " + "_objc_load_image not found"); + + std::vector<char *> Paths; + Paths.resize(RegObjBases.size()); + _objc_map_images(RegObjBases.size(), Paths.data(), + reinterpret_cast<mach_header **>(RegObjBases.data())); + + for (void *RegObjBase : RegObjBases) + _objc_load_image(nullptr, reinterpret_cast<mach_header *>(RegObjBase)); + + return Error::success(); +} + +Error MachOPlatformRuntimeState::runModInits( + std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { + std::vector<span<void (*)()>> InitSections; + InitSections.reserve(JDS.ModInitsSections.numNewSections()); + + // Copy initializer sections: If the JITDylib is unsealed then the + // initializers could reach back into the JIT and cause more initializers to + // be added. + // FIXME: Skip unlock and run in-place on sealed JITDylibs? + JDS.ModInitsSections.processNewSections( + [&](span<void (*)()> Inits) { InitSections.push_back(Inits); }); + + JDStatesLock.unlock(); + for (auto InitSec : InitSections) + for (auto *Init : InitSec) + Init(); + JDStatesLock.lock(); + + return Error::success(); +} + +Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path, + int Mode) { + std::unique_lock<std::mutex> Lock(JDStatesMutex); + + // Try to find JITDylib state by name. + auto *JDS = getJITDylibStateByName(Path); + + if (!JDS) + return make_error<StringError>("No registered JTIDylib for path " + + std::string(Path.data(), Path.size())); + + // If this JITDylib is unsealed, or this is the first dlopen then run + // full dlopen path (update deps, push and run initializers, update ref + // counts on all JITDylibs in the dep tree). + if (!JDS->referenced() || !JDS->Sealed) { + if (auto Err = dlopenFull(Lock, *JDS)) + return std::move(Err); + } + + // Bump the ref-count on this dylib. + ++JDS->DlRefCount; + + // Return the header address. + return JDS->Header; +} + +Error MachOPlatformRuntimeState::dlopenFull( + std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { + // Call back to the JIT to push the initializers. + Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap())); + // Unlock so that we can accept the initializer update. + JDStatesLock.unlock(); + if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>( + SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag, + DepInfo, ExecutorAddr::fromPtr(JDS.Header))) + return Err; + JDStatesLock.lock(); + + if (!DepInfo) + return DepInfo.takeError(); + + if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo)) + return Err; + + if (!DepInfo->empty()) { + ORC_RT_DEBUG({ + printdbg("Unrecognized dep-info key headers in dlopen of %s\n", + JDS.Name.c_str()); + }); + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep-info key headers " + "while processing dlopen of " + << JDS.Name; + return make_error<StringError>(ErrStream.str()); + } + + return Error::success(); +} + +Error MachOPlatformRuntimeState::dlopenInitialize( + std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS, + MachOJITDylibDepInfoMap &DepInfo) { + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + // If the header is not present in the dep map then assume that we + // already processed it earlier in the dlopenInitialize traversal and + // return. + // TODO: Keep a visited set instead so that we can error out on missing + // entries? + auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header)); + if (I == DepInfo.end()) + return Error::success(); + + auto DI = std::move(I->second); + DepInfo.erase(I); + + // We don't need to re-initialize sealed JITDylibs that have already been + // initialized. Just check that their dep-map entry is empty as expected. + if (JDS.Sealed) { + if (!DI.DepHeaders.empty()) { + std::ostringstream ErrStream; + ErrStream << "Sealed JITDylib " << JDS.Header + << " already has registered dependencies"; + return make_error<StringError>(ErrStream.str()); + } + if (JDS.referenced()) + return Error::success(); + } else + JDS.Sealed = DI.Sealed; + + // This is an unsealed or newly sealed JITDylib. Run initializers. + std::vector<JITDylibState *> OldDeps; + std::swap(JDS.Deps, OldDeps); + JDS.Deps.reserve(DI.DepHeaders.size()); + for (auto DepHeaderAddr : DI.DepHeaders) { + auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>()); + if (!DepJDS) { + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep header " + << DepHeaderAddr.toPtr<void *>() << " while initializing " + << JDS.Name; + return make_error<StringError>(ErrStream.str()); + } + ++DepJDS->LinkedAgainstRefCount; + if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo)) + return Err; + } + + // Initialize this JITDylib. + if (auto Err = registerObjCRegistrationObjects(JDS)) + return Err; + if (auto Err = runModInits(JDStatesLock, JDS)) + return Err; + + // Decrement old deps. + // FIXME: We should probably continue and just report deinitialize errors + // here. + for (auto *DepJDS : OldDeps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS)) + return Err; + } + + return Error::success(); +} + +Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { + std::unique_lock<std::mutex> Lock(JDStatesMutex); + + // Try to find JITDylib state by header. + auto *JDS = getJITDylibStateByHeader(DSOHandle); + + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "No registered JITDylib for " << DSOHandle; + return make_error<StringError>(ErrStream.str()); + } + + // Bump the ref-count. + --JDS->DlRefCount; + + if (!JDS->referenced()) + return dlcloseDeinitialize(Lock, *JDS); + + return Error::success(); +} + +Error MachOPlatformRuntimeState::dlcloseDeinitialize( + std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { + + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + runAtExits(JDStatesLock, JDS); + + // Reset mod-inits + JDS.ModInitsSections.reset(); + + // Reset data section contents. + for (auto &KV : JDS.DataSectionContent) + memcpy(KV.first, KV.second.data(), KV.second.size()); + for (auto &KV : JDS.ZeroInitRanges) + memset(KV.first, 0, KV.second); + + // Deinitialize any dependencies. + for (auto *DepJDS : JDS.Deps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS)) + return Err; + } + + return Error::success(); +} + +class MachOPlatformRuntimeTLVManager { +public: + void *getInstance(const char *ThreadData); + +private: + std::unordered_map<const char *, char *> Instances; + std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; +}; + +void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { + auto I = Instances.find(ThreadData); + if (I != Instances.end()) + return I->second; + + auto TDS = + MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); + if (!TDS) { + __orc_rt_log_error(toString(TDS.takeError()).c_str()); + return nullptr; + } + + auto &Allocated = AllocatedSections[TDS->first]; + if (!Allocated) { + Allocated = std::make_unique<char[]>(TDS->second); + memcpy(Allocated.get(), TDS->first, TDS->second); + } + + size_t ThreadDataDelta = ThreadData - TDS->first; + assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); + + char *Instance = Allocated.get() + ThreadDataDelta; + Instances[ThreadData] = Instance; + return Instance; +} + +void destroyMachOTLVMgr(void *MachOTLVMgr) { + delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr); +} + +Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) { + for (auto &WFC : WFCs) + if (auto Err = WFC.runWithSPSRet<void>()) + return Err; + return Error::success(); +} + +} // end anonymous namespace + +//------------------------------------------------------------------------------ +// JIT entry points +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError()>::handle( + ArgData, ArgSize, + []() { return MachOPlatformRuntimeState::create(); }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError()>::handle( + ArgData, ArgSize, + []() { return MachOPlatformRuntimeState::destroy(); }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle( + ArgData, ArgSize, + [](std::string &Name, ExecutorAddr HeaderAddr) { + return MachOPlatformRuntimeState::get().registerJITDylib( + std::move(Name), HeaderAddr.toPtr<void *>()); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSExecutorAddr)>::handle( + ArgData, ArgSize, + [](ExecutorAddr HeaderAddr) { + return MachOPlatformRuntimeState::get().deregisterJITDylib( + HeaderAddr.toPtr<void *>()); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_register_object_platform_sections(char *ArgData, + size_t ArgSize) { + return WrapperFunction<SPSError(SPSExecutorAddr, + SPSOptional<SPSUnwindSectionInfo>, + SPSMachOObjectPlatformSectionsMap)>:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> + &Secs) { + return MachOPlatformRuntimeState::get() + .registerObjectPlatformSections(HeaderAddr, std::move(USI), + std::move(Secs)); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) { + using SymtabContainer = std::vector< + std::tuple<ExecutorAddr, ExecutorAddr, + MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>; + return WrapperFunction<SPSError( + SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr, + SPSMachOExecutorSymbolFlags>>)>:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) { + return MachOPlatformRuntimeState::get() + .registerObjectSymbolTable(HeaderAddr, Symbols); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) { + using SymtabContainer = std::vector< + std::tuple<ExecutorAddr, ExecutorAddr, + MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>; + return WrapperFunction<SPSError( + SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr, + SPSMachOExecutorSymbolFlags>>)>:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) { + return MachOPlatformRuntimeState::get() + .deregisterObjectSymbolTable(HeaderAddr, Symbols); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_deregister_object_platform_sections(char *ArgData, + size_t ArgSize) { + return WrapperFunction<SPSError(SPSExecutorAddr, + SPSOptional<SPSUnwindSectionInfo>, + SPSMachOObjectPlatformSectionsMap)>:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI, + std::vector<std::pair<std::string_view, ExecutorAddrRange>> + &Secs) { + return MachOPlatformRuntimeState::get() + .deregisterObjectPlatformSections(HeaderAddr, std::move(USI), + std::move(Secs)); + }) + .release(); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle( + ArgData, ArgSize, runWrapperFunctionCalls) + .release(); +} + +//------------------------------------------------------------------------------ +// TLV support +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) { + auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>( + pthread_getspecific(D->Key)); + if (!TLVMgr) { + TLVMgr = new MachOPlatformRuntimeTLVManager(); + if (pthread_setspecific(D->Key, TLVMgr)) { + __orc_rt_log_error("Call to pthread_setspecific failed"); + return nullptr; + } + } + + return TLVMgr->getInstance( + reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); +} + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( + ArgData, ArgSize, + []() -> Expected<uint64_t> { + pthread_key_t Key; + if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) { + __orc_rt_log_error("Call to pthread_key_create failed"); + return make_error<StringError>(strerror(Err)); + } + return static_cast<uint64_t>(Key); + }) + .release(); +} + +//------------------------------------------------------------------------------ +// cxa_atexit support +//------------------------------------------------------------------------------ + +int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle) { + return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle); +} + +void __orc_rt_macho_cxa_finalize(void *dso_handle) { + MachOPlatformRuntimeState::get().runAtExits(dso_handle); +} + +//------------------------------------------------------------------------------ +// JIT'd dlfcn alternatives. +//------------------------------------------------------------------------------ + +const char *__orc_rt_macho_jit_dlerror() { + return MachOPlatformRuntimeState::get().dlerror(); +} + +void *__orc_rt_macho_jit_dlopen(const char *path, int mode) { + return MachOPlatformRuntimeState::get().dlopen(path, mode); +} + +int __orc_rt_macho_jit_dlclose(void *dso_handle) { + return MachOPlatformRuntimeState::get().dlclose(dso_handle); +} + +void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) { + return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol); +} + +//------------------------------------------------------------------------------ +// MachO Run Program +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName, + const char *EntrySymbolName, + int argc, char *argv[]) { + using MainTy = int (*)(int, char *[]); + + void *H = __orc_rt_macho_jit_dlopen(JITDylibName, + __orc_rt::macho::ORC_RT_RTLD_LAZY); + if (!H) { + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + return -1; + } + + auto *Main = + reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName)); + + if (!Main) { + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + return -1; + } + + int Result = Main(argc, argv); + + if (__orc_rt_macho_jit_dlclose(H) == -1) + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + + return Result; +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/macho_platform.h b/contrib/llvm-project/compiler-rt/lib/orc/macho_platform.h new file mode 100644 index 000000000000..3b2242ab27ce --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/macho_platform.h @@ -0,0 +1,44 @@ +//===- macho_platform.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC Runtime support for Darwin dynamic loading features. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_MACHO_PLATFORM_H +#define ORC_RT_MACHO_PLATFORM_H + +#include "common.h" +#include "executor_address.h" + +// Atexit functions. +ORC_RT_INTERFACE int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle); +ORC_RT_INTERFACE void __orc_rt_macho_cxa_finalize(void *dso_handle); + +// dlfcn functions. +ORC_RT_INTERFACE const char *__orc_rt_macho_jit_dlerror(); +ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlopen(const char *path, int mode); +ORC_RT_INTERFACE int __orc_rt_macho_jit_dlclose(void *dso_handle); +ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlsym(void *dso_handle, + const char *symbol); + +namespace __orc_rt { +namespace macho { + +enum dlopen_mode : int { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 +}; + +} // end namespace macho +} // end namespace __orc_rt + +#endif // ORC_RT_MACHO_PLATFORM_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/macho_tlv.arm64.S b/contrib/llvm-project/compiler-rt/lib/orc/macho_tlv.arm64.S new file mode 100644 index 000000000000..f6eb9fc4da39 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/macho_tlv.arm64.S @@ -0,0 +1,92 @@ +//===-- macho_tlv.arm64.s ---------------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +// The content of this file is arm64-only +#if defined(__arm64__) || defined(__aarch64__) + +#define REGISTER_SAVE_SPACE_SIZE 32 * 24 + + .text + + // returns address of TLV in x0, all other registers preserved + .globl ___orc_rt_macho_tlv_get_addr +___orc_rt_macho_tlv_get_addr: + sub sp, sp, #REGISTER_SAVE_SPACE_SIZE + stp x29, x30, [sp, #16 * 1] + stp x27, x28, [sp, #16 * 2] + stp x25, x26, [sp, #16 * 3] + stp x23, x24, [sp, #16 * 4] + stp x21, x22, [sp, #16 * 5] + stp x19, x20, [sp, #16 * 6] + stp x17, x18, [sp, #16 * 7] + stp x15, x16, [sp, #16 * 8] + stp x13, x14, [sp, #16 * 9] + stp x11, x12, [sp, #16 * 10] + stp x9, x10, [sp, #16 * 11] + stp x7, x8, [sp, #16 * 12] + stp x5, x6, [sp, #16 * 13] + stp x3, x4, [sp, #16 * 14] + stp x1, x2, [sp, #16 * 15] + stp q30, q31, [sp, #32 * 8] + stp q28, q29, [sp, #32 * 9] + stp q26, q27, [sp, #32 * 10] + stp q24, q25, [sp, #32 * 11] + stp q22, q23, [sp, #32 * 12] + stp q20, q21, [sp, #32 * 13] + stp q18, q19, [sp, #32 * 14] + stp q16, q17, [sp, #32 * 15] + stp q14, q15, [sp, #32 * 16] + stp q12, q13, [sp, #32 * 17] + stp q10, q11, [sp, #32 * 18] + stp q8, q9, [sp, #32 * 19] + stp q6, q7, [sp, #32 * 20] + stp q4, q5, [sp, #32 * 21] + stp q2, q3, [sp, #32 * 22] + stp q0, q1, [sp, #32 * 23] + + bl ___orc_rt_macho_tlv_get_addr_impl + + ldp q0, q1, [sp, #32 * 23] + ldp q2, q3, [sp, #32 * 22] + ldp q4, q5, [sp, #32 * 21] + ldp q6, q7, [sp, #32 * 20] + ldp q8, q9, [sp, #32 * 19] + ldp q10, q11, [sp, #32 * 18] + ldp q12, q13, [sp, #32 * 17] + ldp q14, q15, [sp, #32 * 16] + ldp q16, q17, [sp, #32 * 15] + ldp q18, q19, [sp, #32 * 14] + ldp q20, q21, [sp, #32 * 13] + ldp q22, q23, [sp, #32 * 12] + ldp q24, q25, [sp, #32 * 11] + ldp q26, q27, [sp, #32 * 10] + ldp q28, q29, [sp, #32 * 9] + ldp q30, q31, [sp, #32 * 8] + ldp x1, x2, [sp, #16 * 15] + ldp x3, x4, [sp, #16 * 14] + ldp x5, x6, [sp, #16 * 13] + ldp x7, x8, [sp, #16 * 12] + ldp x9, x10, [sp, #16 * 11] + ldp x11, x12, [sp, #16 * 10] + ldp x13, x14, [sp, #16 * 9] + ldp x15, x16, [sp, #16 * 8] + ldp x17, x18, [sp, #16 * 7] + ldp x19, x20, [sp, #16 * 6] + ldp x21, x22, [sp, #16 * 5] + ldp x23, x24, [sp, #16 * 4] + ldp x25, x26, [sp, #16 * 3] + ldp x27, x28, [sp, #16 * 2] + ldp x29, x30, [sp, #16 * 1] + add sp, sp, #REGISTER_SAVE_SPACE_SIZE + ret + +#endif // defined(__arm64__) || defined(__aarch64__) diff --git a/contrib/llvm-project/compiler-rt/lib/orc/macho_tlv.x86-64.S b/contrib/llvm-project/compiler-rt/lib/orc/macho_tlv.x86-64.S new file mode 100644 index 000000000000..e3daf23e3029 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/macho_tlv.x86-64.S @@ -0,0 +1,73 @@ +//===-- orc_rt_macho_tlv.x86-64.s -------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +// The content of this file is x86_64-only +#if defined(__x86_64__) + +#define REGISTER_SAVE_SPACE_SIZE 512 + + .text + + // returns address of TLV in %rax, all other registers preserved + .globl ___orc_rt_macho_tlv_get_addr +___orc_rt_macho_tlv_get_addr: + pushq %rbp + movq %rsp, %rbp + subq $REGISTER_SAVE_SPACE_SIZE, %rsp + movq %rbx, -8(%rbp) + movq %rcx, -16(%rbp) + movq %rdx, -24(%rbp) + movq %rsi, -32(%rbp) + movq %rdi, -40(%rbp) + movq %r8, -48(%rbp) + movq %r9, -56(%rbp) + movq %r10, -64(%rbp) + movq %r11, -72(%rbp) + movq %r12, -80(%rbp) + movq %r13, -88(%rbp) + movq %r14, -96(%rbp) + movq %r15, -104(%rbp) + movdqa %xmm0, -128(%rbp) + movdqa %xmm1, -144(%rbp) + movdqa %xmm2, -160(%rbp) + movdqa %xmm3, -176(%rbp) + movdqa %xmm4, -192(%rbp) + movdqa %xmm5, -208(%rbp) + movdqa %xmm6, -224(%rbp) + movdqa %xmm7, -240(%rbp) + call ___orc_rt_macho_tlv_get_addr_impl + movq -8(%rbp), %rbx + movq -16(%rbp), %rcx + movq -24(%rbp), %rdx + movq -32(%rbp), %rsi + movq -40(%rbp), %rdi + movq -48(%rbp), %r8 + movq -56(%rbp), %r9 + movq -64(%rbp), %r10 + movq -72(%rbp), %r11 + movq -80(%rbp), %r12 + movq -88(%rbp), %r13 + movq -96(%rbp), %r14 + movq -104(%rbp), %r15 + movdqa -128(%rbp), %xmm0 + movdqa -144(%rbp), %xmm1 + movdqa -160(%rbp), %xmm2 + movdqa -176(%rbp), %xmm3 + movdqa -192(%rbp), %xmm4 + movdqa -208(%rbp), %xmm5 + movdqa -224(%rbp), %xmm6 + movdqa -240(%rbp), %xmm7 + addq $REGISTER_SAVE_SPACE_SIZE, %rsp + popq %rbp + ret + +#endif // defined(__x86_64__) diff --git a/contrib/llvm-project/compiler-rt/lib/orc/run_program_wrapper.cpp b/contrib/llvm-project/compiler-rt/lib/orc/run_program_wrapper.cpp new file mode 100644 index 000000000000..24a7b4fc3cbe --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/run_program_wrapper.cpp @@ -0,0 +1,51 @@ +//===- run_program_wrapper.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "adt.h" +#include "common.h" +#include "wrapper_function_utils.h" + +#include <vector> + +using namespace __orc_rt; + +extern "C" int64_t __orc_rt_run_program(const char *JITDylibName, + const char *EntrySymbolName, int argc, + char *argv[]); + +ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult +__orc_rt_run_program_wrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<int64_t(SPSString, SPSString, + SPSSequence<SPSString>)>:: + handle(ArgData, ArgSize, + [](const std::string &JITDylibName, + const std::string &EntrySymbolName, + const std::vector<std::string_view> &Args) { + std::vector<std::unique_ptr<char[]>> ArgVStorage; + ArgVStorage.reserve(Args.size()); + for (auto &Arg : Args) { + ArgVStorage.push_back( + std::make_unique<char[]>(Arg.size() + 1)); + memcpy(ArgVStorage.back().get(), Arg.data(), Arg.size()); + ArgVStorage.back()[Arg.size()] = '\0'; + } + std::vector<char *> ArgV; + ArgV.reserve(ArgVStorage.size() + 1); + for (auto &ArgStorage : ArgVStorage) + ArgV.push_back(ArgStorage.get()); + ArgV.push_back(nullptr); + return __orc_rt_run_program(JITDylibName.c_str(), + EntrySymbolName.c_str(), + ArgV.size() - 1, ArgV.data()); + }) + .release(); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/simple_packed_serialization.h b/contrib/llvm-project/compiler-rt/lib/orc/simple_packed_serialization.h new file mode 100644 index 000000000000..488d2407ddd4 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/simple_packed_serialization.h @@ -0,0 +1,689 @@ +//===--- simple_packed_serialization.h - simple serialization ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +// The behavior of the utilities in this header must be synchronized with the +// behavior of the utilities in +// llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h. +// +// The Simple Packed Serialization (SPS) utilities are used to generate +// argument and return buffers for wrapper functions using the following +// serialization scheme: +// +// Primitives: +// bool, char, int8_t, uint8_t -- Two's complement 8-bit (0=false, 1=true) +// int16_t, uint16_t -- Two's complement 16-bit little endian +// int32_t, uint32_t -- Two's complement 32-bit little endian +// int64_t, int64_t -- Two's complement 64-bit little endian +// +// Sequence<T>: +// Serialized as the sequence length (as a uint64_t) followed by the +// serialization of each of the elements without padding. +// +// Tuple<T1, ..., TN>: +// Serialized as each of the element types from T1 to TN without padding. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_SIMPLE_PACKED_SERIALIZATION_H +#define ORC_RT_SIMPLE_PACKED_SERIALIZATION_H + +#include "adt.h" +#include "endianness.h" +#include "error.h" +#include "stl_extras.h" + +#include <optional> +#include <string> +#include <string_view> +#include <tuple> +#include <type_traits> +#include <unordered_map> +#include <utility> +#include <vector> + +namespace __orc_rt { + +/// Output char buffer with overflow check. +class SPSOutputBuffer { +public: + SPSOutputBuffer(char *Buffer, size_t Remaining) + : Buffer(Buffer), Remaining(Remaining) {} + bool write(const char *Data, size_t Size) { + if (Size > Remaining) + return false; + memcpy(Buffer, Data, Size); + Buffer += Size; + Remaining -= Size; + return true; + } + +private: + char *Buffer = nullptr; + size_t Remaining = 0; +}; + +/// Input char buffer with underflow check. +class SPSInputBuffer { +public: + SPSInputBuffer() = default; + SPSInputBuffer(const char *Buffer, size_t Remaining) + : Buffer(Buffer), Remaining(Remaining) {} + bool read(char *Data, size_t Size) { + if (Size > Remaining) + return false; + memcpy(Data, Buffer, Size); + Buffer += Size; + Remaining -= Size; + return true; + } + + const char *data() const { return Buffer; } + bool skip(size_t Size) { + if (Size > Remaining) + return false; + Buffer += Size; + Remaining -= Size; + return true; + } + +private: + const char *Buffer = nullptr; + size_t Remaining = 0; +}; + +/// Specialize to describe how to serialize/deserialize to/from the given +/// concrete type. +template <typename SPSTagT, typename ConcreteT, typename _ = void> +class SPSSerializationTraits; + +/// A utility class for serializing to a blob from a variadic list. +template <typename... ArgTs> class SPSArgList; + +// Empty list specialization for SPSArgList. +template <> class SPSArgList<> { +public: + static size_t size() { return 0; } + + static bool serialize(SPSOutputBuffer &OB) { return true; } + static bool deserialize(SPSInputBuffer &IB) { return true; } +}; + +// Non-empty list specialization for SPSArgList. +template <typename SPSTagT, typename... SPSTagTs> +class SPSArgList<SPSTagT, SPSTagTs...> { +public: + template <typename ArgT, typename... ArgTs> + static size_t size(const ArgT &Arg, const ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::size(Arg) + + SPSArgList<SPSTagTs...>::size(Args...); + } + + template <typename ArgT, typename... ArgTs> + static bool serialize(SPSOutputBuffer &OB, const ArgT &Arg, + const ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::serialize(OB, Arg) && + SPSArgList<SPSTagTs...>::serialize(OB, Args...); + } + + template <typename ArgT, typename... ArgTs> + static bool deserialize(SPSInputBuffer &IB, ArgT &Arg, ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::deserialize(IB, Arg) && + SPSArgList<SPSTagTs...>::deserialize(IB, Args...); + } +}; + +/// SPS serialization for integral types, bool, and char. +template <typename SPSTagT> +class SPSSerializationTraits< + SPSTagT, SPSTagT, + std::enable_if_t<std::is_same<SPSTagT, bool>::value || + std::is_same<SPSTagT, char>::value || + std::is_same<SPSTagT, int8_t>::value || + std::is_same<SPSTagT, int16_t>::value || + std::is_same<SPSTagT, int32_t>::value || + std::is_same<SPSTagT, int64_t>::value || + std::is_same<SPSTagT, uint8_t>::value || + std::is_same<SPSTagT, uint16_t>::value || + std::is_same<SPSTagT, uint32_t>::value || + std::is_same<SPSTagT, uint64_t>::value>> { +public: + static size_t size(const SPSTagT &Value) { return sizeof(SPSTagT); } + + static bool serialize(SPSOutputBuffer &OB, const SPSTagT &Value) { + SPSTagT Tmp = Value; + if (IsBigEndianHost) + swapByteOrder(Tmp); + return OB.write(reinterpret_cast<const char *>(&Tmp), sizeof(Tmp)); + } + + static bool deserialize(SPSInputBuffer &IB, SPSTagT &Value) { + SPSTagT Tmp; + if (!IB.read(reinterpret_cast<char *>(&Tmp), sizeof(Tmp))) + return false; + if (IsBigEndianHost) + swapByteOrder(Tmp); + Value = Tmp; + return true; + } +}; + +/// Any empty placeholder suitable as a substitute for void when deserializing +class SPSEmpty {}; + +/// Represents an address in the executor. +class SPSExecutorAddr {}; + +/// SPS tag type for tuples. +/// +/// A blob tuple should be serialized by serializing each of the elements in +/// sequence. +template <typename... SPSTagTs> class SPSTuple { +public: + /// Convenience typedef of the corresponding arg list. + typedef SPSArgList<SPSTagTs...> AsArgList; +}; + +/// SPS tag type for optionals. +/// +/// SPSOptionals should be serialized as a bool with true indicating that an +/// SPSTagT value is present, and false indicating that there is no value. +/// If the boolean is true then the serialized SPSTagT will follow immediately +/// after it. +template <typename SPSTagT> class SPSOptional {}; + +/// SPS tag type for sequences. +/// +/// SPSSequences should be serialized as a uint64_t sequence length, +/// followed by the serialization of each of the elements. +template <typename SPSElementTagT> class SPSSequence; + +/// SPS tag type for strings, which are equivalent to sequences of chars. +using SPSString = SPSSequence<char>; + +/// SPS tag type for maps. +/// +/// SPS maps are just sequences of (Key, Value) tuples. +template <typename SPSTagT1, typename SPSTagT2> +using SPSMap = SPSSequence<SPSTuple<SPSTagT1, SPSTagT2>>; + +/// Serialization for SPSEmpty type. +template <> class SPSSerializationTraits<SPSEmpty, SPSEmpty> { +public: + static size_t size(const SPSEmpty &EP) { return 0; } + static bool serialize(SPSOutputBuffer &OB, const SPSEmpty &BE) { + return true; + } + static bool deserialize(SPSInputBuffer &IB, SPSEmpty &BE) { return true; } +}; + +/// Specialize this to implement 'trivial' sequence serialization for +/// a concrete sequence type. +/// +/// Trivial sequence serialization uses the sequence's 'size' member to get the +/// length of the sequence, and uses a range-based for loop to iterate over the +/// elements. +/// +/// Specializing this template class means that you do not need to provide a +/// specialization of SPSSerializationTraits for your type. +template <typename SPSElementTagT, typename ConcreteSequenceT> +class TrivialSPSSequenceSerialization { +public: + static constexpr bool available = false; +}; + +/// Specialize this to implement 'trivial' sequence deserialization for +/// a concrete sequence type. +/// +/// Trivial deserialization calls a static 'reserve(SequenceT&)' method on your +/// specialization (you must implement this) to reserve space, and then calls +/// a static 'append(SequenceT&, ElementT&) method to append each of the +/// deserialized elements. +/// +/// Specializing this template class means that you do not need to provide a +/// specialization of SPSSerializationTraits for your type. +template <typename SPSElementTagT, typename ConcreteSequenceT> +class TrivialSPSSequenceDeserialization { +public: + static constexpr bool available = false; +}; + +/// Trivial std::string -> SPSSequence<char> serialization. +template <> class TrivialSPSSequenceSerialization<char, std::string> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<char> -> std::string deserialization. +template <> class TrivialSPSSequenceDeserialization<char, std::string> { +public: + static constexpr bool available = true; + + using element_type = char; + + static void reserve(std::string &S, uint64_t Size) { S.reserve(Size); } + static bool append(std::string &S, char C) { + S.push_back(C); + return true; + } +}; + +/// Trivial std::vector<T> -> SPSSequence<SPSElementTagT> serialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceSerialization<SPSElementTagT, std::vector<T>> { +public: + static constexpr bool available = true; +}; + +/// Trivial span<T> -> SPSSequence<SPSElementTagT> serialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceSerialization<SPSElementTagT, span<T>> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<SPSElementTagT> -> std::vector<T> deserialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceDeserialization<SPSElementTagT, std::vector<T>> { +public: + static constexpr bool available = true; + + using element_type = typename std::vector<T>::value_type; + + static void reserve(std::vector<T> &V, uint64_t Size) { V.reserve(Size); } + static bool append(std::vector<T> &V, T E) { + V.push_back(std::move(E)); + return true; + } +}; + +/// Trivial std::unordered_map<K, V> -> SPSSequence<SPSTuple<SPSKey, SPSValue>> +/// serialization. +template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V> +class TrivialSPSSequenceSerialization<SPSTuple<SPSKeyTagT, SPSValueTagT>, + std::unordered_map<K, V>> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<SPSTuple<SPSKey, SPSValue>> -> std::unordered_map<K, V> +/// deserialization. +template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V> +class TrivialSPSSequenceDeserialization<SPSTuple<SPSKeyTagT, SPSValueTagT>, + std::unordered_map<K, V>> { +public: + static constexpr bool available = true; + + using element_type = std::pair<K, V>; + + static void reserve(std::unordered_map<K, V> &M, uint64_t Size) { + M.reserve(Size); + } + static bool append(std::unordered_map<K, V> &M, element_type E) { + return M.insert(std::move(E)).second; + } +}; + +/// 'Trivial' sequence serialization: Sequence is serialized as a uint64_t size +/// followed by a for-earch loop over the elements of the sequence to serialize +/// each of them. +template <typename SPSElementTagT, typename SequenceT> +class SPSSerializationTraits<SPSSequence<SPSElementTagT>, SequenceT, + std::enable_if_t<TrivialSPSSequenceSerialization< + SPSElementTagT, SequenceT>::available>> { +public: + static size_t size(const SequenceT &S) { + size_t Size = SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())); + for (const auto &E : S) + Size += SPSArgList<SPSElementTagT>::size(E); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, const SequenceT &S) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size()))) + return false; + for (const auto &E : S) + if (!SPSArgList<SPSElementTagT>::serialize(OB, E)) + return false; + return true; + } + + static bool deserialize(SPSInputBuffer &IB, SequenceT &S) { + using TBSD = TrivialSPSSequenceDeserialization<SPSElementTagT, SequenceT>; + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + TBSD::reserve(S, Size); + for (size_t I = 0; I != Size; ++I) { + typename TBSD::element_type E; + if (!SPSArgList<SPSElementTagT>::deserialize(IB, E)) + return false; + if (!TBSD::append(S, std::move(E))) + return false; + } + return true; + } +}; + +/// Trivial serialization / deserialization for span<char> +template <> class SPSSerializationTraits<SPSSequence<char>, span<const char>> { +public: + static size_t size(const span<const char> &S) { + return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) + + S.size(); + } + static bool serialize(SPSOutputBuffer &OB, const span<const char> &S) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size()))) + return false; + return OB.write(S.data(), S.size()); + } + static bool deserialize(SPSInputBuffer &IB, span<const char> &S) { + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + S = span<const char>(IB.data(), Size); + return IB.skip(Size); + } +}; + +/// SPSTuple serialization for std::tuple. +template <typename... SPSTagTs, typename... Ts> +class SPSSerializationTraits<SPSTuple<SPSTagTs...>, std::tuple<Ts...>> { +private: + using TupleArgList = typename SPSTuple<SPSTagTs...>::AsArgList; + using ArgIndices = std::make_index_sequence<sizeof...(Ts)>; + + template <std::size_t... I> + static size_t size(const std::tuple<Ts...> &T, std::index_sequence<I...>) { + return TupleArgList::size(std::get<I>(T)...); + } + + template <std::size_t... I> + static bool serialize(SPSOutputBuffer &OB, const std::tuple<Ts...> &T, + std::index_sequence<I...>) { + return TupleArgList::serialize(OB, std::get<I>(T)...); + } + + template <std::size_t... I> + static bool deserialize(SPSInputBuffer &IB, std::tuple<Ts...> &T, + std::index_sequence<I...>) { + return TupleArgList::deserialize(IB, std::get<I>(T)...); + } + +public: + static size_t size(const std::tuple<Ts...> &T) { + return size(T, ArgIndices{}); + } + + static bool serialize(SPSOutputBuffer &OB, const std::tuple<Ts...> &T) { + return serialize(OB, T, ArgIndices{}); + } + + static bool deserialize(SPSInputBuffer &IB, std::tuple<Ts...> &T) { + return deserialize(IB, T, ArgIndices{}); + } +}; + +/// SPSTuple serialization for std::pair. +template <typename SPSTagT1, typename SPSTagT2, typename T1, typename T2> +class SPSSerializationTraits<SPSTuple<SPSTagT1, SPSTagT2>, std::pair<T1, T2>> { +public: + static size_t size(const std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::size(P.first) + + SPSArgList<SPSTagT2>::size(P.second); + } + + static bool serialize(SPSOutputBuffer &OB, const std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::serialize(OB, P.first) && + SPSArgList<SPSTagT2>::serialize(OB, P.second); + } + + static bool deserialize(SPSInputBuffer &IB, std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::deserialize(IB, P.first) && + SPSArgList<SPSTagT2>::deserialize(IB, P.second); + } +}; + +/// SPSOptional serialization for std::optional. +template <typename SPSTagT, typename T> +class SPSSerializationTraits<SPSOptional<SPSTagT>, std::optional<T>> { +public: + static size_t size(const std::optional<T> &Value) { + size_t Size = SPSArgList<bool>::size(!!Value); + if (Value) + Size += SPSArgList<SPSTagT>::size(*Value); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, const std::optional<T> &Value) { + if (!SPSArgList<bool>::serialize(OB, !!Value)) + return false; + if (Value) + return SPSArgList<SPSTagT>::serialize(OB, *Value); + return true; + } + + static bool deserialize(SPSInputBuffer &IB, std::optional<T> &Value) { + bool HasValue; + if (!SPSArgList<bool>::deserialize(IB, HasValue)) + return false; + if (HasValue) { + Value = T(); + return SPSArgList<SPSTagT>::deserialize(IB, *Value); + } else + Value = std::optional<T>(); + return true; + } +}; + +/// Serialization for string_views. +/// +/// Serialization is as for regular strings. Deserialization points directly +/// into the blob. +template <> class SPSSerializationTraits<SPSString, std::string_view> { +public: + static size_t size(const std::string_view &S) { + return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) + + S.size(); + } + + static bool serialize(SPSOutputBuffer &OB, const std::string_view &S) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size()))) + return false; + return OB.write(S.data(), S.size()); + } + + static bool deserialize(SPSInputBuffer &IB, std::string_view &S) { + const char *Data = nullptr; + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + if (Size > std::numeric_limits<size_t>::max()) + return false; + Data = IB.data(); + if (!IB.skip(Size)) + return false; + S = {Data, static_cast<size_t>(Size)}; + return true; + } +}; + +/// SPS tag type for errors. +class SPSError; + +/// SPS tag type for expecteds, which are either a T or a string representing +/// an error. +template <typename SPSTagT> class SPSExpected; + +namespace detail { + +/// Helper type for serializing Errors. +/// +/// llvm::Errors are move-only, and not inspectable except by consuming them. +/// This makes them unsuitable for direct serialization via +/// SPSSerializationTraits, which needs to inspect values twice (once to +/// determine the amount of space to reserve, and then again to serialize). +/// +/// The SPSSerializableError type is a helper that can be +/// constructed from an llvm::Error, but inspected more than once. +struct SPSSerializableError { + bool HasError = false; + std::string ErrMsg; +}; + +/// Helper type for serializing Expected<T>s. +/// +/// See SPSSerializableError for more details. +/// +// FIXME: Use std::variant for storage once we have c++17. +template <typename T> struct SPSSerializableExpected { + bool HasValue = false; + T Value{}; + std::string ErrMsg; +}; + +inline SPSSerializableError toSPSSerializable(Error Err) { + if (Err) + return {true, toString(std::move(Err))}; + return {false, {}}; +} + +inline Error fromSPSSerializable(SPSSerializableError BSE) { + if (BSE.HasError) + return make_error<StringError>(BSE.ErrMsg); + return Error::success(); +} + +template <typename T> +SPSSerializableExpected<T> toSPSSerializable(Expected<T> E) { + if (E) + return {true, std::move(*E), {}}; + else + return {false, {}, toString(E.takeError())}; +} + +template <typename T> +Expected<T> fromSPSSerializable(SPSSerializableExpected<T> BSE) { + if (BSE.HasValue) + return std::move(BSE.Value); + else + return make_error<StringError>(BSE.ErrMsg); +} + +} // end namespace detail + +/// Serialize to a SPSError from a detail::SPSSerializableError. +template <> +class SPSSerializationTraits<SPSError, detail::SPSSerializableError> { +public: + static size_t size(const detail::SPSSerializableError &BSE) { + size_t Size = SPSArgList<bool>::size(BSE.HasError); + if (BSE.HasError) + Size += SPSArgList<SPSString>::size(BSE.ErrMsg); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableError &BSE) { + if (!SPSArgList<bool>::serialize(OB, BSE.HasError)) + return false; + if (BSE.HasError) + if (!SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg)) + return false; + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + detail::SPSSerializableError &BSE) { + if (!SPSArgList<bool>::deserialize(IB, BSE.HasError)) + return false; + + if (!BSE.HasError) + return true; + + return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a +/// detail::SPSSerializableExpected<T>. +template <typename SPSTagT, typename T> +class SPSSerializationTraits<SPSExpected<SPSTagT>, + detail::SPSSerializableExpected<T>> { +public: + static size_t size(const detail::SPSSerializableExpected<T> &BSE) { + size_t Size = SPSArgList<bool>::size(BSE.HasValue); + if (BSE.HasValue) + Size += SPSArgList<SPSTagT>::size(BSE.Value); + else + Size += SPSArgList<SPSString>::size(BSE.ErrMsg); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableExpected<T> &BSE) { + if (!SPSArgList<bool>::serialize(OB, BSE.HasValue)) + return false; + + if (BSE.HasValue) + return SPSArgList<SPSTagT>::serialize(OB, BSE.Value); + + return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg); + } + + static bool deserialize(SPSInputBuffer &IB, + detail::SPSSerializableExpected<T> &BSE) { + if (!SPSArgList<bool>::deserialize(IB, BSE.HasValue)) + return false; + + if (BSE.HasValue) + return SPSArgList<SPSTagT>::deserialize(IB, BSE.Value); + + return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a detail::SPSSerializableError. +template <typename SPSTagT> +class SPSSerializationTraits<SPSExpected<SPSTagT>, + detail::SPSSerializableError> { +public: + static size_t size(const detail::SPSSerializableError &BSE) { + assert(BSE.HasError && "Cannot serialize expected from a success value"); + return SPSArgList<bool>::size(false) + + SPSArgList<SPSString>::size(BSE.ErrMsg); + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableError &BSE) { + assert(BSE.HasError && "Cannot serialize expected from a success value"); + if (!SPSArgList<bool>::serialize(OB, false)) + return false; + return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a T. +template <typename SPSTagT, typename T> +class SPSSerializationTraits<SPSExpected<SPSTagT>, T> { +public: + static size_t size(const T &Value) { + return SPSArgList<bool>::size(true) + SPSArgList<SPSTagT>::size(Value); + } + + static bool serialize(SPSOutputBuffer &OB, const T &Value) { + if (!SPSArgList<bool>::serialize(OB, true)) + return false; + return SPSArgList<SPSTagT>::serialize(Value); + } +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_SIMPLE_PACKED_SERIALIZATION_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/stl_extras.h b/contrib/llvm-project/compiler-rt/lib/orc/stl_extras.h new file mode 100644 index 000000000000..80a6cd13ac28 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/stl_extras.h @@ -0,0 +1,45 @@ +//===-------- stl_extras.h - Useful STL related functions-------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_STL_EXTRAS_H +#define ORC_RT_STL_EXTRAS_H + +#include <cstdint> +#include <utility> +#include <tuple> + +namespace __orc_rt { + +/// Substitute for std::identity. +/// Switch to std::identity once we can use c++20. +template <class Ty> struct identity { + using is_transparent = void; + using argument_type = Ty; + + Ty &operator()(Ty &self) const { return self; } + const Ty &operator()(const Ty &self) const { return self; } +}; + +/// Substitute for std::bit_ceil. +constexpr uint64_t bit_ceil(uint64_t Val) noexcept { + Val |= (Val >> 1); + Val |= (Val >> 2); + Val |= (Val >> 4); + Val |= (Val >> 8); + Val |= (Val >> 16); + Val |= (Val >> 32); + return Val + 1; +} + +} // namespace __orc_rt + +#endif // ORC_RT_STL_EXTRAS diff --git a/contrib/llvm-project/compiler-rt/lib/orc/string_pool.h b/contrib/llvm-project/compiler-rt/lib/orc/string_pool.h new file mode 100644 index 000000000000..c0ba4ea8980e --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/string_pool.h @@ -0,0 +1,172 @@ +//===------- string_pool.h - Thread-safe pool for strings -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains a thread-safe string pool. Strings are ref-counted, but not +// automatically deallocated. Unused entries can be cleared by calling +// StringPool::clearDeadEntries. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_STRING_POOL_H +#define ORC_RT_STRING_POOL_H + +#include <atomic> +#include <cassert> +#include <functional> +#include <mutex> +#include <string> +#include <unordered_map> + +namespace __orc_rt { + +class PooledStringPtr; + +/// String pool for strings names used by the ORC runtime. +class StringPool { + friend class PooledStringPtr; + +public: + /// Destroy a StringPool. + ~StringPool(); + + /// Create a string pointer from the given string. + PooledStringPtr intern(std::string S); + + /// Remove from the pool any entries that are no longer referenced. + void clearDeadEntries(); + + /// Returns true if the pool is empty. + bool empty() const; + +private: + using RefCountType = std::atomic<size_t>; + using PoolMap = std::unordered_map<std::string, RefCountType>; + using PoolMapEntry = PoolMap::value_type; + mutable std::mutex PoolMutex; + PoolMap Pool; +}; + +/// Pointer to a pooled string. +class PooledStringPtr { + friend class StringPool; + friend struct std::hash<PooledStringPtr>; + +public: + PooledStringPtr() = default; + PooledStringPtr(std::nullptr_t) {} + PooledStringPtr(const PooledStringPtr &Other) : S(Other.S) { + if (S) + ++S->second; + } + + PooledStringPtr &operator=(const PooledStringPtr &Other) { + if (S) { + assert(S->second && "Releasing PooledStringPtr with zero ref count"); + --S->second; + } + S = Other.S; + if (S) + ++S->second; + return *this; + } + + PooledStringPtr(PooledStringPtr &&Other) : S(nullptr) { + std::swap(S, Other.S); + } + + PooledStringPtr &operator=(PooledStringPtr &&Other) { + if (S) { + assert(S->second && "Releasing PooledStringPtr with zero ref count"); + --S->second; + } + S = nullptr; + std::swap(S, Other.S); + return *this; + } + + ~PooledStringPtr() { + if (S) { + assert(S->second && "Releasing PooledStringPtr with zero ref count"); + --S->second; + } + } + + explicit operator bool() const { return S; } + + const std::string &operator*() const { return S->first; } + + friend bool operator==(const PooledStringPtr &LHS, + const PooledStringPtr &RHS) { + return LHS.S == RHS.S; + } + + friend bool operator!=(const PooledStringPtr &LHS, + const PooledStringPtr &RHS) { + return !(LHS == RHS); + } + + friend bool operator<(const PooledStringPtr &LHS, + const PooledStringPtr &RHS) { + return LHS.S < RHS.S; + } + +private: + using PoolEntry = StringPool::PoolMapEntry; + using PoolEntryPtr = PoolEntry *; + + PooledStringPtr(StringPool::PoolMapEntry *S) : S(S) { + if (S) + ++S->second; + } + + PoolEntryPtr S = nullptr; +}; + +inline StringPool::~StringPool() { +#ifndef NDEBUG + clearDeadEntries(); + assert(Pool.empty() && "Dangling references at pool destruction time"); +#endif // NDEBUG +} + +inline PooledStringPtr StringPool::intern(std::string S) { + std::lock_guard<std::mutex> Lock(PoolMutex); + PoolMap::iterator I; + bool Added; + std::tie(I, Added) = Pool.try_emplace(std::move(S), 0); + return PooledStringPtr(&*I); +} + +inline void StringPool::clearDeadEntries() { + std::lock_guard<std::mutex> Lock(PoolMutex); + for (auto I = Pool.begin(), E = Pool.end(); I != E;) { + auto Tmp = I++; + if (Tmp->second == 0) + Pool.erase(Tmp); + } +} + +inline bool StringPool::empty() const { + std::lock_guard<std::mutex> Lock(PoolMutex); + return Pool.empty(); +} + +} // end namespace __orc_rt + +namespace std { + +// Make PooledStringPtrs hashable. +template <> struct hash<__orc_rt::PooledStringPtr> { + size_t operator()(const __orc_rt::PooledStringPtr &A) const { + return hash<__orc_rt::PooledStringPtr::PoolEntryPtr>()(A.S); + } +}; + +} // namespace std + +#endif // ORC_RT_REF_COUNTED_STRING_POOL_H diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/tools/orc-rt-executor.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/tools/orc-rt-executor.cpp new file mode 100644 index 000000000000..da45a2d64d68 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/tools/orc-rt-executor.cpp @@ -0,0 +1,49 @@ +//===- orc-rt-executor.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the orc-rt-executor test tool. This is a "blank executable" that +// links the ORC runtime and can accept code from a JIT controller like lii or +// llvm-jitlink. +// +//===----------------------------------------------------------------------===// + +#include <cstring> +#include <iostream> +#include <optional> +#include <string_view> + +void printHelp(std::string_view ProgName, std::ostream &OS) { + OS << "usage: " << ProgName << " [help] [<mode>] <program arguments>...\n" + << " <mode> -- specify how to listen for JIT'd program\n" + << " filedesc=<in>,<out> -- read from <in> filedesc, write to out\n" + << " tcp=<host>:<port> -- listen on the given host/port\n" + << " help -- print help and exit\n" + << "\n" + << " Notes:\n" + << " Program arguments will be made available to the JIT controller.\n" + << " When running a JIT'd program containing a main function the\n" + << " controller may choose to pass these on to main, however\n" + << " orc-rt-executor does not enforce this.\n"; +} + +int main(int argc, char *argv[]) { + if (argc < 2) { + printHelp("orc-rt-executor", std::cerr); + std::cerr << "error: insufficient arguments.\n"; + exit(1); + } + + if (!strcmp(argv[1], "help")) { + printHelp(argv[0], std::cerr); + exit(0); + } + + std::cerr << "error: One day I will be a real program, but I am not yet.\n"; + + return 0; +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/adt_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/adt_test.cpp new file mode 100644 index 000000000000..6625a590e363 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/adt_test.cpp @@ -0,0 +1,50 @@ +//===-- adt_test.cpp ------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#include "adt.h" +#include "gtest/gtest.h" + +#include <sstream> +#include <string> + +using namespace __orc_rt; + +TEST(ADTTest, SpanDefaultConstruction) { + span<int> S; + EXPECT_TRUE(S.empty()) << "Default constructed span not empty"; + EXPECT_EQ(S.size(), 0U) << "Default constructed span size not zero"; + EXPECT_EQ(S.begin(), S.end()) << "Default constructed span begin != end"; +} + +TEST(ADTTest, SpanConstructFromFixedArray) { + int A[] = {1, 2, 3, 4, 5}; + span<int> S(A); + EXPECT_FALSE(S.empty()) << "Span should be non-empty"; + EXPECT_EQ(S.size(), 5U) << "Span has unexpected size"; + EXPECT_EQ(std::distance(S.begin(), S.end()), 5U) + << "Unexpected iterator range size"; + EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value"; + for (unsigned I = 0; I != S.size(); ++I) + EXPECT_EQ(S[I], A[I]) << "Unexpected span element value"; +} + +TEST(ADTTest, SpanConstructFromIteratorAndSize) { + int A[] = {1, 2, 3, 4, 5}; + span<int> S(&A[0], 5); + EXPECT_FALSE(S.empty()) << "Span should be non-empty"; + EXPECT_EQ(S.size(), 5U) << "Span has unexpected size"; + EXPECT_EQ(std::distance(S.begin(), S.end()), 5U) + << "Unexpected iterator range size"; + EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value"; + for (unsigned I = 0; I != S.size(); ++I) + EXPECT_EQ(S[I], A[I]) << "Unexpected span element value"; +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/bitmask_enum_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/bitmask_enum_test.cpp new file mode 100644 index 000000000000..4c27d54fb4a9 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/bitmask_enum_test.cpp @@ -0,0 +1,143 @@ +//===-- adt_test.cpp ------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#include "bitmask_enum.h" +#include "gtest/gtest.h" + +#include <sstream> +#include <string> + +using namespace __orc_rt; + +namespace { + +enum Flags { F0 = 0, F1 = 1, F2 = 2, F3 = 4, F4 = 8 }; + +} // namespace + +namespace __orc_rt { +ORC_RT_DECLARE_ENUM_AS_BITMASK(Flags, F4); +} // namespace __orc_rt + +static_assert(is_bitmask_enum<Flags>::value != 0); +static_assert(largest_bitmask_enum_bit<Flags>::value == Flags::F4); + +namespace { + +static_assert(is_bitmask_enum<Flags>::value != 0); +static_assert(largest_bitmask_enum_bit<Flags>::value == Flags::F4); + +TEST(BitmaskEnumTest, BitwiseOr) { + Flags f = F1 | F2; + EXPECT_EQ(3, f); + + f = f | F3; + EXPECT_EQ(7, f); +} + +TEST(BitmaskEnumTest, BitwiseOrEquals) { + Flags f = F1; + f |= F3; + EXPECT_EQ(5, f); + + // |= should return a reference to the LHS. + f = F2; + (f |= F3) = F1; + EXPECT_EQ(F1, f); +} + +TEST(BitmaskEnumTest, BitwiseAnd) { + Flags f = static_cast<Flags>(3) & F2; + EXPECT_EQ(F2, f); + + f = (f | F3) & (F1 | F2 | F3); + EXPECT_EQ(6, f); +} + +TEST(BitmaskEnumTest, BitwiseAndEquals) { + Flags f = F1 | F2 | F3; + f &= F1 | F2; + EXPECT_EQ(3, f); + + // &= should return a reference to the LHS. + (f &= F1) = F3; + EXPECT_EQ(F3, f); +} + +TEST(BitmaskEnumTest, BitwiseXor) { + Flags f = (F1 | F2) ^ (F2 | F3); + EXPECT_EQ(5, f); + + f = f ^ F1; + EXPECT_EQ(4, f); +} + +TEST(BitmaskEnumTest, BitwiseXorEquals) { + Flags f = (F1 | F2); + f ^= (F2 | F4); + EXPECT_EQ(9, f); + + // ^= should return a reference to the LHS. + (f ^= F4) = F3; + EXPECT_EQ(F3, f); +} + +TEST(BitmaskEnumTest, ConstantExpression) { + constexpr Flags f1 = ~F1; + constexpr Flags f2 = F1 | F2; + constexpr Flags f3 = F1 & F2; + constexpr Flags f4 = F1 ^ F2; + EXPECT_EQ(f1, ~F1); + EXPECT_EQ(f2, F1 | F2); + EXPECT_EQ(f3, F1 & F2); + EXPECT_EQ(f4, F1 ^ F2); +} + +TEST(BitmaskEnumTest, BitwiseNot) { + Flags f = ~F1; + EXPECT_EQ(14, f); // Largest value for f is 15. + EXPECT_EQ(15, ~F0); +} + +enum class FlagsClass { + F0 = 0, + F1 = 1, + F2 = 2, + F3 = 4, + ORC_RT_MARK_AS_BITMASK_ENUM(F3) +}; + +TEST(BitmaskEnumTest, ScopedEnum) { + FlagsClass f = (FlagsClass::F1 & ~FlagsClass::F0) | FlagsClass::F2; + f |= FlagsClass::F3; + EXPECT_EQ(7, static_cast<int>(f)); +} + +struct Container { + enum Flags { + F0 = 0, + F1 = 1, + F2 = 2, + F3 = 4, + ORC_RT_MARK_AS_BITMASK_ENUM(F3) + }; + + static Flags getFlags() { + Flags f = F0 | F1; + f |= F2; + return f; + } +}; + +TEST(BitmaskEnumTest, EnumInStruct) { EXPECT_EQ(3, Container::getFlags()); } + +} // namespace diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/c_api_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/c_api_test.cpp new file mode 100644 index 000000000000..497cb937e2af --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/c_api_test.cpp @@ -0,0 +1,200 @@ +//===-- c_api_test.cpp ----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#include "orc_rt/c_api.h" +#include "gtest/gtest.h" + +TEST(CAPITest, CWrapperFunctionResultInit) { + orc_rt_CWrapperFunctionResult R; + orc_rt_CWrapperFunctionResultInit(&R); + + EXPECT_EQ(R.Size, 0U); + EXPECT_EQ(R.Data.ValuePtr, nullptr); + + // Check that this value isn't treated as an out-of-band error. + EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr); + + // Check that we can dispose of the value. + orc_rt_DisposeCWrapperFunctionResult(&R); +} + +TEST(CAPITest, CWrapperFunctionResultAllocSmall) { + constexpr size_t SmallAllocSize = sizeof(const char *); + + auto R = orc_rt_CWrapperFunctionResultAllocate(SmallAllocSize); + char *DataPtr = orc_rt_CWrapperFunctionResultData(&R); + + for (size_t I = 0; I != SmallAllocSize; ++I) + DataPtr[I] = 0x55 + I; + + // Check that the inline storage in R.Data.Value contains the expected + // sequence. + EXPECT_EQ(R.Size, SmallAllocSize); + for (size_t I = 0; I != SmallAllocSize; ++I) + EXPECT_EQ(R.Data.Value[I], (char)(0x55 + I)) + << "Unexpected value at index " << I; + + // Check that this value isn't treated as an out-of-band error. + EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr); + + // Check that orc_rt_CWrapperFunctionResult(Data|Result|Size) and + // orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected. + EXPECT_EQ(orc_rt_CWrapperFunctionResultData(&R), R.Data.Value); + EXPECT_EQ(orc_rt_CWrapperFunctionResultSize(&R), SmallAllocSize); + EXPECT_FALSE(orc_rt_CWrapperFunctionResultEmpty(&R)); + EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr); + + // Check that we can dispose of the value. + orc_rt_DisposeCWrapperFunctionResult(&R); +} + +TEST(CAPITest, CWrapperFunctionResultAllocLarge) { + constexpr size_t LargeAllocSize = sizeof(const char *) + 1; + + auto R = orc_rt_CWrapperFunctionResultAllocate(LargeAllocSize); + char *DataPtr = orc_rt_CWrapperFunctionResultData(&R); + + for (size_t I = 0; I != LargeAllocSize; ++I) + DataPtr[I] = 0x55 + I; + + // Check that the inline storage in R.Data.Value contains the expected + // sequence. + EXPECT_EQ(R.Size, LargeAllocSize); + EXPECT_EQ(R.Data.ValuePtr, DataPtr); + for (size_t I = 0; I != LargeAllocSize; ++I) + EXPECT_EQ(R.Data.ValuePtr[I], (char)(0x55 + I)) + << "Unexpected value at index " << I; + + // Check that this value isn't treated as an out-of-band error. + EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr); + + // Check that orc_rt_CWrapperFunctionResult(Data|Result|Size) and + // orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected. + EXPECT_EQ(orc_rt_CWrapperFunctionResultData(&R), R.Data.ValuePtr); + EXPECT_EQ(orc_rt_CWrapperFunctionResultSize(&R), LargeAllocSize); + EXPECT_FALSE(orc_rt_CWrapperFunctionResultEmpty(&R)); + EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr); + + // Check that we can dispose of the value. + orc_rt_DisposeCWrapperFunctionResult(&R); +} + +TEST(CAPITest, CWrapperFunctionResultFromRangeSmall) { + constexpr size_t SmallAllocSize = sizeof(const char *); + + char Source[SmallAllocSize]; + for (size_t I = 0; I != SmallAllocSize; ++I) + Source[I] = 0x55 + I; + + orc_rt_CWrapperFunctionResult R = + orc_rt_CreateCWrapperFunctionResultFromRange(Source, SmallAllocSize); + + // Check that the inline storage in R.Data.Value contains the expected + // sequence. + EXPECT_EQ(R.Size, SmallAllocSize); + for (size_t I = 0; I != SmallAllocSize; ++I) + EXPECT_EQ(R.Data.Value[I], (char)(0x55 + I)) + << "Unexpected value at index " << I; + + // Check that we can dispose of the value. + orc_rt_DisposeCWrapperFunctionResult(&R); +} + +TEST(CAPITest, CWrapperFunctionResultFromRangeLarge) { + constexpr size_t LargeAllocSize = sizeof(const char *) + 1; + + char Source[LargeAllocSize]; + for (size_t I = 0; I != LargeAllocSize; ++I) + Source[I] = 0x55 + I; + + orc_rt_CWrapperFunctionResult R = + orc_rt_CreateCWrapperFunctionResultFromRange(Source, LargeAllocSize); + + // Check that the inline storage in R.Data.Value contains the expected + // sequence. + EXPECT_EQ(R.Size, LargeAllocSize); + for (size_t I = 0; I != LargeAllocSize; ++I) + EXPECT_EQ(R.Data.ValuePtr[I], (char)(0x55 + I)) + << "Unexpected value at index " << I; + + // Check that we can dispose of the value. + orc_rt_DisposeCWrapperFunctionResult(&R); +} + +TEST(CAPITest, CWrapperFunctionResultFromStringSmall) { + constexpr size_t SmallAllocSize = sizeof(const char *); + + char Source[SmallAllocSize]; + for (size_t I = 0; I != SmallAllocSize - 1; ++I) + Source[I] = 'a' + I; + Source[SmallAllocSize - 1] = '\0'; + + orc_rt_CWrapperFunctionResult R = + orc_rt_CreateCWrapperFunctionResultFromString(Source); + + // Check that the inline storage in R.Data.Value contains the expected + // sequence. + EXPECT_EQ(R.Size, SmallAllocSize); + for (size_t I = 0; I != SmallAllocSize - 1; ++I) + EXPECT_EQ(R.Data.Value[I], (char)('a' + I)) + << "Unexpected value at index " << I; + EXPECT_EQ(R.Data.Value[SmallAllocSize - 1], '\0') + << "Unexpected value at index " << (SmallAllocSize - 1); + + // Check that we can dispose of the value. + orc_rt_DisposeCWrapperFunctionResult(&R); +} + +TEST(CAPITest, CWrapperFunctionResultFromStringLarge) { + constexpr size_t LargeAllocSize = sizeof(const char *) + 1; + + char Source[LargeAllocSize]; + for (size_t I = 0; I != LargeAllocSize - 1; ++I) + Source[I] = 'a' + I; + Source[LargeAllocSize - 1] = '\0'; + + orc_rt_CWrapperFunctionResult R = + orc_rt_CreateCWrapperFunctionResultFromString(Source); + + // Check that the inline storage in R.Data.Value contains the expected + // sequence. + EXPECT_EQ(R.Size, LargeAllocSize); + for (size_t I = 0; I != LargeAllocSize - 1; ++I) + EXPECT_EQ(R.Data.ValuePtr[I], (char)('a' + I)) + << "Unexpected value at index " << I; + EXPECT_EQ(R.Data.ValuePtr[LargeAllocSize - 1], '\0') + << "Unexpected value at index " << (LargeAllocSize - 1); + + // Check that we can dispose of the value. + orc_rt_DisposeCWrapperFunctionResult(&R); +} + +TEST(CAPITest, CWrapperFunctionResultFromOutOfBandError) { + constexpr const char *ErrMsg = "test error message"; + orc_rt_CWrapperFunctionResult R = + orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(ErrMsg); + +#ifndef NDEBUG + EXPECT_DEATH({ orc_rt_CWrapperFunctionResultData(&R); }, + "Cannot get data for out-of-band error value"); + EXPECT_DEATH({ orc_rt_CWrapperFunctionResultSize(&R); }, + "Cannot get size for out-of-band error value"); +#endif + + EXPECT_FALSE(orc_rt_CWrapperFunctionResultEmpty(&R)); + const char *OOBErrMsg = orc_rt_CWrapperFunctionResultGetOutOfBandError(&R); + EXPECT_NE(OOBErrMsg, nullptr); + EXPECT_NE(OOBErrMsg, ErrMsg); + EXPECT_TRUE(strcmp(OOBErrMsg, ErrMsg) == 0); + + orc_rt_DisposeCWrapperFunctionResult(&R); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/endian_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/endian_test.cpp new file mode 100644 index 000000000000..71b677af694c --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/endian_test.cpp @@ -0,0 +1,174 @@ +//===- endian_test.cpp ------------------------- swap byte order test -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +// Adapted from the llvm/unittests/Support/SwapByteOrderTest.cpp LLVM unit test. +// +//===----------------------------------------------------------------------===// + +#include "endianness.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +TEST(Endian, ByteSwap_32) { + EXPECT_EQ(0x44332211u, ByteSwap_32(0x11223344)); + EXPECT_EQ(0xDDCCBBAAu, ByteSwap_32(0xAABBCCDD)); +} + +TEST(Endian, ByteSwap_64) { + EXPECT_EQ(0x8877665544332211ULL, ByteSwap_64(0x1122334455667788LL)); + EXPECT_EQ(0x1100FFEEDDCCBBAAULL, ByteSwap_64(0xAABBCCDDEEFF0011LL)); +} + +// In these first two tests all of the original_uintx values are truncated +// except for 64. We could avoid this, but there's really no point. +TEST(Endian, getSwappedBytes_UnsignedRoundTrip) { + // The point of the bit twiddling of magic is to test with and without bits + // in every byte. + uint64_t value = 1; + for (std::size_t i = 0; i <= sizeof(value); ++i) { + uint8_t original_uint8 = static_cast<uint8_t>(value); + EXPECT_EQ(original_uint8, getSwappedBytes(getSwappedBytes(original_uint8))); + + uint16_t original_uint16 = static_cast<uint16_t>(value); + EXPECT_EQ(original_uint16, + getSwappedBytes(getSwappedBytes(original_uint16))); + + uint32_t original_uint32 = static_cast<uint32_t>(value); + EXPECT_EQ(original_uint32, + getSwappedBytes(getSwappedBytes(original_uint32))); + + uint64_t original_uint64 = static_cast<uint64_t>(value); + EXPECT_EQ(original_uint64, + getSwappedBytes(getSwappedBytes(original_uint64))); + + value = (value << 8) | 0x55; // binary 0101 0101. + } +} + +TEST(Endian, getSwappedBytes_SignedRoundTrip) { + // The point of the bit twiddling of magic is to test with and without bits + // in every byte. + uint64_t value = 1; + for (std::size_t i = 0; i <= sizeof(value); ++i) { + int8_t original_int8 = static_cast<int8_t>(value); + EXPECT_EQ(original_int8, getSwappedBytes(getSwappedBytes(original_int8))); + + int16_t original_int16 = static_cast<int16_t>(value); + EXPECT_EQ(original_int16, getSwappedBytes(getSwappedBytes(original_int16))); + + int32_t original_int32 = static_cast<int32_t>(value); + EXPECT_EQ(original_int32, getSwappedBytes(getSwappedBytes(original_int32))); + + int64_t original_int64 = static_cast<int64_t>(value); + EXPECT_EQ(original_int64, getSwappedBytes(getSwappedBytes(original_int64))); + + // Test other sign. + value *= -1; + + original_int8 = static_cast<int8_t>(value); + EXPECT_EQ(original_int8, getSwappedBytes(getSwappedBytes(original_int8))); + + original_int16 = static_cast<int16_t>(value); + EXPECT_EQ(original_int16, getSwappedBytes(getSwappedBytes(original_int16))); + + original_int32 = static_cast<int32_t>(value); + EXPECT_EQ(original_int32, getSwappedBytes(getSwappedBytes(original_int32))); + + original_int64 = static_cast<int64_t>(value); + EXPECT_EQ(original_int64, getSwappedBytes(getSwappedBytes(original_int64))); + + // Return to normal sign and twiddle. + value *= -1; + value = (value << 8) | 0x55; // binary 0101 0101. + } +} + +TEST(Endian, getSwappedBytes_uint8_t) { + EXPECT_EQ(uint8_t(0x11), getSwappedBytes(uint8_t(0x11))); +} + +TEST(Endian, getSwappedBytes_uint16_t) { + EXPECT_EQ(uint16_t(0x1122), getSwappedBytes(uint16_t(0x2211))); +} + +TEST(Endian, getSwappedBytes_uint32_t) { + EXPECT_EQ(uint32_t(0x11223344), getSwappedBytes(uint32_t(0x44332211))); +} + +TEST(Endian, getSwappedBytes_uint64_t) { + EXPECT_EQ(uint64_t(0x1122334455667788ULL), + getSwappedBytes(uint64_t(0x8877665544332211ULL))); +} + +TEST(Endian, getSwappedBytes_int8_t) { + EXPECT_EQ(int8_t(0x11), getSwappedBytes(int8_t(0x11))); +} + +TEST(Endian, getSwappedBytes_int16_t) { + EXPECT_EQ(int16_t(0x1122), getSwappedBytes(int16_t(0x2211))); +} + +TEST(Endian, getSwappedBytes_int32_t) { + EXPECT_EQ(int32_t(0x11223344), getSwappedBytes(int32_t(0x44332211))); +} + +TEST(Endian, getSwappedBytes_int64_t) { + EXPECT_EQ(int64_t(0x1122334455667788LL), + getSwappedBytes(int64_t(0x8877665544332211LL))); +} + +TEST(Endian, swapByteOrder_uint8_t) { + uint8_t value = 0x11; + swapByteOrder(value); + EXPECT_EQ(uint8_t(0x11), value); +} + +TEST(Endian, swapByteOrder_uint16_t) { + uint16_t value = 0x2211; + swapByteOrder(value); + EXPECT_EQ(uint16_t(0x1122), value); +} + +TEST(Endian, swapByteOrder_uint32_t) { + uint32_t value = 0x44332211; + swapByteOrder(value); + EXPECT_EQ(uint32_t(0x11223344), value); +} + +TEST(Endian, swapByteOrder_uint64_t) { + uint64_t value = 0x8877665544332211ULL; + swapByteOrder(value); + EXPECT_EQ(uint64_t(0x1122334455667788ULL), value); +} + +TEST(Endian, swapByteOrder_int8_t) { + int8_t value = 0x11; + swapByteOrder(value); + EXPECT_EQ(int8_t(0x11), value); +} + +TEST(Endian, swapByteOrder_int16_t) { + int16_t value = 0x2211; + swapByteOrder(value); + EXPECT_EQ(int16_t(0x1122), value); +} + +TEST(Endian, swapByteOrder_int32_t) { + int32_t value = 0x44332211; + swapByteOrder(value); + EXPECT_EQ(int32_t(0x11223344), value); +} + +TEST(Endian, swapByteOrder_int64_t) { + int64_t value = 0x8877665544332211LL; + swapByteOrder(value); + EXPECT_EQ(int64_t(0x1122334455667788LL), value); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/error_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/error_test.cpp new file mode 100644 index 000000000000..5251d788e01b --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/error_test.cpp @@ -0,0 +1,295 @@ +//===-- error_test.cpp --sssssssss-----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +// Note: +// This unit test was adapted from +// llvm/unittests/Support/ExtensibleRTTITest.cpp +// +//===----------------------------------------------------------------------===// + +#include "error.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +namespace { + +class CustomError : public RTTIExtends<CustomError, ErrorInfoBase> { +public: + CustomError(int V1) : V1(V1) {} + std::string toString() const override { + return "CustomError V1 = " + std::to_string(V1); + } + int getV1() const { return V1; } + +protected: + int V1; +}; + +class CustomSubError : public RTTIExtends<CustomSubError, CustomError> { +public: + CustomSubError(int V1, std::string V2) + : RTTIExtends<CustomSubError, CustomError>(V1), V2(std::move(V2)) {} + std::string toString() const override { + return "CustomSubError V1 = " + std::to_string(V1) + ", " + V2; + } + const std::string &getV2() const { return V2; } + +protected: + std::string V2; +}; + +} // end anonymous namespace + +// Test that a checked success value doesn't cause any issues. +TEST(Error, CheckedSuccess) { + Error E = Error::success(); + EXPECT_FALSE(E) << "Unexpected error while testing Error 'Success'"; +} + +// Check that a consumed success value doesn't cause any issues. +TEST(Error, ConsumeSuccess) { consumeError(Error::success()); } + +TEST(Error, ConsumeError) { + Error E = make_error<CustomError>(42); + if (E) { + consumeError(std::move(E)); + } else + ADD_FAILURE() << "Error failure value should convert to true"; +} + +// Test that unchecked success values cause an abort. +TEST(Error, UncheckedSuccess) { + EXPECT_DEATH({ Error E = Error::success(); }, + "Error must be checked prior to destruction") + << "Unchecked Error Succes value did not cause abort()"; +} + +// Test that a checked but unhandled error causes an abort. +TEST(Error, CheckedButUnhandledError) { + auto DropUnhandledError = []() { + Error E = make_error<CustomError>(42); + (void)!E; + }; + EXPECT_DEATH(DropUnhandledError(), + "Error must be checked prior to destruction") + << "Unhandled Error failure value did not cause an abort()"; +} + +// Test that error_cast works as expected. +TEST(Error, BasicErrorCast) { + { + // Check casting base error value to base error type. + auto E = make_error<CustomError>(42); + if (auto CSE = error_cast<CustomSubError>(E)) { + ADD_FAILURE() << "Derived cast incorrectly matched base error"; + } else if (auto CE = error_cast<CustomError>(E)) { + EXPECT_EQ(CE->getV1(), 42) << "Unexpected wrapped value"; + } else + ADD_FAILURE() << "Unexpected error value"; + } + + { + // Check casting derived error value to base error type. + auto E = make_error<CustomSubError>(42, "foo"); + if (auto CE = error_cast<CustomError>(E)) { + EXPECT_EQ(CE->getV1(), 42) << "Unexpected wrapped value"; + } else + ADD_FAILURE() << "Unexpected error value"; + } + + { + // Check casting derived error value to derived error type. + auto E = make_error<CustomSubError>(42, "foo"); + if (auto CSE = error_cast<CustomSubError>(E)) { + EXPECT_EQ(CSE->getV1(), 42) << "Unexpected wrapped value"; + EXPECT_EQ(CSE->getV2(), "foo") << "Unexpected wrapped value"; + } else + ADD_FAILURE() << "Unexpected error value"; + } +} + +// ErrorAsOutParameter tester. +static void errAsOutParamHelper(Error &Err) { + ErrorAsOutParameter ErrAsOutParam(&Err); + // Verify that checked flag is raised - assignment should not crash. + Err = Error::success(); + // Raise the checked bit manually - caller should still have to test the + // error. + (void)!!Err; +} + +// Test that ErrorAsOutParameter sets the checked flag on construction. +TEST(Error, ErrorAsOutParameterChecked) { + Error E = Error::success(); + errAsOutParamHelper(E); + (void)!!E; +} + +// Test that ErrorAsOutParameter clears the checked flag on destruction. +TEST(Error, ErrorAsOutParameterUnchecked) { + EXPECT_DEATH( + { + Error E = Error::success(); + errAsOutParamHelper(E); + }, + "Error must be checked prior to destruction") + << "ErrorAsOutParameter did not clear the checked flag on destruction."; +} + +// Check 'Error::isA<T>' method handling. +TEST(Error, IsAHandling) { + // Check 'isA' handling. + Error E = make_error<CustomError>(42); + Error F = make_error<CustomSubError>(42, "foo"); + Error G = Error::success(); + + EXPECT_TRUE(E.isA<CustomError>()); + EXPECT_FALSE(E.isA<CustomSubError>()); + EXPECT_TRUE(F.isA<CustomError>()); + EXPECT_TRUE(F.isA<CustomSubError>()); + EXPECT_FALSE(G.isA<CustomError>()); + + consumeError(std::move(E)); + consumeError(std::move(F)); + consumeError(std::move(G)); +} + +TEST(Error, StringError) { + auto E = make_error<StringError>("foo"); + if (auto SE = error_cast<StringError>(E)) { + EXPECT_EQ(SE->toString(), "foo") << "Unexpected StringError value"; + } else + ADD_FAILURE() << "Expected StringError value"; +} + +// Test Checked Expected<T> in success mode. +TEST(Error, CheckedExpectedInSuccessMode) { + Expected<int> A = 7; + EXPECT_TRUE(!!A) << "Expected with non-error value doesn't convert to 'true'"; + // Access is safe in second test, since we checked the error in the first. + EXPECT_EQ(*A, 7) << "Incorrect Expected non-error value"; +} + +// Test Expected with reference type. +TEST(Error, ExpectedWithReferenceType) { + int A = 7; + Expected<int &> B = A; + // 'Check' B. + (void)!!B; + int &C = *B; + EXPECT_EQ(&A, &C) << "Expected failed to propagate reference"; +} + +// Test Unchecked Expected<T> in success mode. +// We expect this to blow up the same way Error would. +// Test runs in debug mode only. +TEST(Error, UncheckedExpectedInSuccessModeDestruction) { + EXPECT_DEATH({ Expected<int> A = 7; }, + "Expected<T> must be checked before access or destruction.") + << "Unchecekd Expected<T> success value did not cause an abort()."; +} + +// Test Unchecked Expected<T> in success mode. +// We expect this to blow up the same way Error would. +// Test runs in debug mode only. +TEST(Error, UncheckedExpectedInSuccessModeAccess) { + EXPECT_DEATH( + { + Expected<int> A = 7; + *A; + }, + "Expected<T> must be checked before access or destruction.") + << "Unchecekd Expected<T> success value did not cause an abort()."; +} + +// Test Unchecked Expected<T> in success mode. +// We expect this to blow up the same way Error would. +// Test runs in debug mode only. +TEST(Error, UncheckedExpectedInSuccessModeAssignment) { + EXPECT_DEATH( + { + Expected<int> A = 7; + A = 7; + }, + "Expected<T> must be checked before access or destruction.") + << "Unchecekd Expected<T> success value did not cause an abort()."; +} + +// Test Expected<T> in failure mode. +TEST(Error, ExpectedInFailureMode) { + Expected<int> A = make_error<CustomError>(42); + EXPECT_FALSE(!!A) << "Expected with error value doesn't convert to 'false'"; + Error E = A.takeError(); + EXPECT_TRUE(E.isA<CustomError>()) << "Incorrect Expected error value"; + consumeError(std::move(E)); +} + +// Check that an Expected instance with an error value doesn't allow access to +// operator*. +// Test runs in debug mode only. +TEST(Error, AccessExpectedInFailureMode) { + Expected<int> A = make_error<CustomError>(42); + EXPECT_DEATH(*A, "Expected<T> must be checked before access or destruction.") + << "Incorrect Expected error value"; + consumeError(A.takeError()); +} + +// Check that an Expected instance with an error triggers an abort if +// unhandled. +// Test runs in debug mode only. +TEST(Error, UnhandledExpectedInFailureMode) { + EXPECT_DEATH({ Expected<int> A = make_error<CustomError>(42); }, + "Expected<T> must be checked before access or destruction.") + << "Unchecked Expected<T> failure value did not cause an abort()"; +} + +// Test covariance of Expected. +TEST(Error, ExpectedCovariance) { + class B {}; + class D : public B {}; + + Expected<B *> A1(Expected<D *>(nullptr)); + // Check A1 by converting to bool before assigning to it. + (void)!!A1; + A1 = Expected<D *>(nullptr); + // Check A1 again before destruction. + (void)!!A1; + + Expected<std::unique_ptr<B>> A2(Expected<std::unique_ptr<D>>(nullptr)); + // Check A2 by converting to bool before assigning to it. + (void)!!A2; + A2 = Expected<std::unique_ptr<D>>(nullptr); + // Check A2 again before destruction. + (void)!!A2; +} + +// Test that the ExitOnError utility works as expected. +TEST(Error, CantFailSuccess) { + cantFail(Error::success()); + + int X = cantFail(Expected<int>(42)); + EXPECT_EQ(X, 42) << "Expected value modified by cantFail"; + + int Dummy = 42; + int &Y = cantFail(Expected<int &>(Dummy)); + EXPECT_EQ(&Dummy, &Y) << "Reference mangled by cantFail"; +} + +// Test that cantFail results in a crash if you pass it a failure value. +TEST(Error, CantFailDeath) { + EXPECT_DEATH(cantFail(make_error<StringError>("foo")), + "cantFail called on failure value") + << "cantFail(Error) did not cause an abort for failure value"; + + EXPECT_DEATH(cantFail(Expected<int>(make_error<StringError>("foo"))), + "cantFail called on failure value") + << "cantFail(Expected<int>) did not cause an abort for failure value"; +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/executor_address_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/executor_address_test.cpp new file mode 100644 index 000000000000..05b91f3f8609 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/executor_address_test.cpp @@ -0,0 +1,115 @@ +//===-- executor_address_test.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +// Note: +// This unit test was adapted from +// llvm/unittests/Support/ExecutorAddressTest.cpp +// +//===----------------------------------------------------------------------===// + +#include "executor_address.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +TEST(ExecutorAddrTest, DefaultAndNull) { + // Check that default constructed values and isNull behave as expected. + + ExecutorAddr Default; + ExecutorAddr Null(0); + ExecutorAddr NonNull(1); + + EXPECT_TRUE(Null.isNull()); + EXPECT_EQ(Default, Null); + + EXPECT_FALSE(NonNull.isNull()); + EXPECT_NE(Default, NonNull); +} + +TEST(ExecutorAddrTest, Ordering) { + // Check that ordering operations. + ExecutorAddr A1(1), A2(2); + + EXPECT_LE(A1, A1); + EXPECT_LT(A1, A2); + EXPECT_GT(A2, A1); + EXPECT_GE(A2, A2); +} + +TEST(ExecutorAddrTest, PtrConversion) { + // Test toPtr / fromPtr round-tripping. + int X = 0; + auto XAddr = ExecutorAddr::fromPtr(&X); + int *XPtr = XAddr.toPtr<int *>(); + + EXPECT_EQ(XPtr, &X); +} + +static void F() {} + +TEST(ExecutorAddrTest, PtrConversionWithFunctionType) { + // Test that function types (as opposed to function pointer types) can be + // used with toPtr. + auto FAddr = ExecutorAddr::fromPtr(F); + void (*FPtr)() = FAddr.toPtr<void()>(); + + EXPECT_EQ(FPtr, &F); +} + +TEST(ExecutorAddrTest, WrappingAndUnwrapping) { + constexpr uintptr_t RawAddr = 0x123456; + int *RawPtr = (int *)RawAddr; + + constexpr uintptr_t TagOffset = 8 * (sizeof(uintptr_t) - 1); + uintptr_t TagVal = 0xA5; + uintptr_t TagBits = TagVal << TagOffset; + void *TaggedPtr = (void *)((uintptr_t)RawPtr | TagBits); + + ExecutorAddr EA = + ExecutorAddr::fromPtr(TaggedPtr, ExecutorAddr::Untag(8, TagOffset)); + + EXPECT_EQ(EA.getValue(), RawAddr); + + void *ReconstitutedTaggedPtr = + EA.toPtr<void *>(ExecutorAddr::Tag(TagVal, TagOffset)); + + EXPECT_EQ(TaggedPtr, ReconstitutedTaggedPtr); +} + +TEST(ExecutorAddrTest, AddrRanges) { + ExecutorAddr A0(0), A1(1), A2(2), A3(3); + ExecutorAddrRange R0(A0, A1), R1(A1, A2), R2(A2, A3), R3(A0, A2), R4(A1, A3); + // 012 + // R0: # -- Before R1 + // R1: # -- + // R2: # -- After R1 + // R3: ## -- Overlaps R1 start + // R4: ## -- Overlaps R1 end + + EXPECT_EQ(R1, ExecutorAddrRange(A1, A2)); + EXPECT_EQ(R1, ExecutorAddrRange(A1, ExecutorAddrDiff(1))); + EXPECT_NE(R1, R2); + + EXPECT_TRUE(R1.contains(A1)); + EXPECT_FALSE(R1.contains(A0)); + EXPECT_FALSE(R1.contains(A2)); + + EXPECT_FALSE(R1.overlaps(R0)); + EXPECT_FALSE(R1.overlaps(R2)); + EXPECT_TRUE(R1.overlaps(R3)); + EXPECT_TRUE(R1.overlaps(R4)); +} + +TEST(ExecutorAddrTest, Hashable) { + uint64_t RawAddr = 0x1234567890ABCDEF; + ExecutorAddr Addr(RawAddr); + + EXPECT_EQ(std::hash<uint64_t>()(RawAddr), std::hash<ExecutorAddr>()(Addr)); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/executor_symbol_def_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/executor_symbol_def_test.cpp new file mode 100644 index 000000000000..181091ca1e60 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/executor_symbol_def_test.cpp @@ -0,0 +1,19 @@ +//===-- executor_symbol_def_test.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "executor_symbol_def.h" +#include "simple_packed_serialization_utils.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +TEST(ExecutorSymbolDefTest, Serialization) { + blobSerializationRoundTrip<SPSExecutorSymbolDef>(ExecutorSymbolDef{}); + blobSerializationRoundTrip<SPSExecutorSymbolDef>( + ExecutorSymbolDef{ExecutorAddr{0x70}, {JITSymbolFlags::Callable, 9}}); +}
\ No newline at end of file diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/extensible_rtti_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/extensible_rtti_test.cpp new file mode 100644 index 000000000000..feca1ec1d18c --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/extensible_rtti_test.cpp @@ -0,0 +1,54 @@ +//===-- extensible_rtti_test.cpp ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +// Note: +// This unit test was adapted from +// llvm/unittests/Support/ExtensibleRTTITest.cpp +// +//===----------------------------------------------------------------------===// + +#include "extensible_rtti.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +namespace { + +class MyBase : public RTTIExtends<MyBase, RTTIRoot> {}; + +class MyDerivedA : public RTTIExtends<MyDerivedA, MyBase> {}; + +class MyDerivedB : public RTTIExtends<MyDerivedB, MyBase> {}; + +} // end anonymous namespace + +TEST(ExtensibleRTTITest, BaseCheck) { + MyBase MB; + MyDerivedA MDA; + MyDerivedB MDB; + + // Check MB properties. + EXPECT_TRUE(isa<RTTIRoot>(MB)); + EXPECT_TRUE(isa<MyBase>(MB)); + EXPECT_FALSE(isa<MyDerivedA>(MB)); + EXPECT_FALSE(isa<MyDerivedB>(MB)); + + // Check MDA properties. + EXPECT_TRUE(isa<RTTIRoot>(MDA)); + EXPECT_TRUE(isa<MyBase>(MDA)); + EXPECT_TRUE(isa<MyDerivedA>(MDA)); + EXPECT_FALSE(isa<MyDerivedB>(MDA)); + + // Check MDB properties. + EXPECT_TRUE(isa<RTTIRoot>(MDB)); + EXPECT_TRUE(isa<MyBase>(MDB)); + EXPECT_FALSE(isa<MyDerivedA>(MDB)); + EXPECT_TRUE(isa<MyDerivedB>(MDB)); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/interval_map_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/interval_map_test.cpp new file mode 100644 index 000000000000..a1c6958fcd52 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/interval_map_test.cpp @@ -0,0 +1,204 @@ +//===-- interval_map_test.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#include "interval_map.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +TEST(IntervalMapTest, DefaultConstructed) { + // Check that a default-constructed IntervalMap behaves as expected. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M; + + EXPECT_TRUE(M.empty()); + EXPECT_TRUE(M.begin() == M.end()); + EXPECT_TRUE(M.find(0) == M.end()); +} + +TEST(IntervalMapTest, InsertSingleElement) { + // Check that a map with a single element inserted behaves as expected. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M; + + M.insert(7, 8, 42); + + EXPECT_FALSE(M.empty()); + EXPECT_EQ(std::next(M.begin()), M.end()); + EXPECT_EQ(M.find(7), M.begin()); + EXPECT_EQ(M.find(8), M.end()); + EXPECT_EQ(M.lookup(7), 42U); + EXPECT_EQ(M.lookup(8), 0U); // 8 not present, so should return unsigned(). +} + +TEST(IntervalMapTest, InsertCoalesceWithPrevious) { + // Check that insertions coalesce with previous ranges that share the same + // value. Also check that they _don't_ coalesce if the values are different. + + // Check that insertion coalesces with previous range when values are equal. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1; + + M1.insert(7, 8, 42); + M1.insert(8, 9, 42); + + EXPECT_FALSE(M1.empty()); + EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range. + EXPECT_EQ(M1.find(7), M1.find(8)); // 7 and 8 should point to same range. + EXPECT_EQ(M1.lookup(7), 42U); // Value should be preserved. + + // Check that insertion does not coalesce with previous range when values are + // not equal. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2; + + M2.insert(7, 8, 42); + M2.insert(8, 9, 7); + + EXPECT_FALSE(M2.empty()); + EXPECT_EQ(std::next(std::next(M2.begin())), M2.end()); // Expect two ranges. + EXPECT_NE(M2.find(7), M2.find(8)); // 7 and 8 should be different ranges. + EXPECT_EQ(M2.lookup(7), 42U); // Keys 7 and 8 should map to different values. + EXPECT_EQ(M2.lookup(8), 7U); +} + +TEST(IntervalMapTest, InsertCoalesceWithFollowing) { + // Check that insertions coalesce with following ranges that share the same + // value. Also check that they _don't_ coalesce if the values are different. + + // Check that insertion coalesces with following range when values are equal. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1; + + M1.insert(8, 9, 42); + M1.insert(7, 8, 42); + + EXPECT_FALSE(M1.empty()); + EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range. + EXPECT_EQ(M1.find(7), M1.find(8)); // 7 and 8 should point to same range. + EXPECT_EQ(M1.lookup(7), 42U); // Value should be preserved. + + // Check that insertion does not coalesce with previous range when values are + // not equal. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2; + + M2.insert(8, 9, 42); + M2.insert(7, 8, 7); + + EXPECT_FALSE(M2.empty()); + EXPECT_EQ(std::next(std::next(M2.begin())), M2.end()); // Expect two ranges. + EXPECT_EQ(M2.lookup(7), 7U); // Keys 7 and 8 should map to different values. + EXPECT_EQ(M2.lookup(8), 42U); +} + +TEST(IntervalMapTest, InsertCoalesceBoth) { + // Check that insertions coalesce with ranges on both sides where posssible. + // Also check that they _don't_ coalesce if the values are different. + + // Check that insertion coalesces with both previous and following ranges + // when values are equal. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1; + + M1.insert(7, 8, 42); + M1.insert(9, 10, 42); + + // Check no coalescing yet. + EXPECT_NE(M1.find(7), M1.find(9)); + + // Insert a 3rd range to trigger coalescing on both sides. + M1.insert(8, 9, 42); + + EXPECT_FALSE(M1.empty()); + EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range. + EXPECT_EQ(M1.find(7), M1.find(8)); // 7, 8, and 9 should point to same range. + EXPECT_EQ(M1.find(8), M1.find(9)); + EXPECT_EQ(M1.lookup(7), 42U); // Value should be preserved. + + // Check that insertion does not coalesce with previous range when values are + // not equal. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2; + + M2.insert(7, 8, 42); + M2.insert(8, 9, 7); + M2.insert(9, 10, 42); + + EXPECT_FALSE(M2.empty()); + // Expect three ranges. + EXPECT_EQ(std::next(std::next(std::next(M2.begin()))), M2.end()); + EXPECT_NE(M2.find(7), M2.find(8)); // All keys should map to different ranges. + EXPECT_NE(M2.find(8), M2.find(9)); + EXPECT_EQ(M2.lookup(7), 42U); // Key 7, 8, and 9 should map to different vals. + EXPECT_EQ(M2.lookup(8), 7U); + EXPECT_EQ(M2.lookup(9), 42U); +} + +TEST(IntervalMapTest, EraseSingleElement) { + // Check that we can insert and then remove a single range. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M; + + M.insert(7, 10, 42); + EXPECT_FALSE(M.empty()); + M.erase(7, 10); + EXPECT_TRUE(M.empty()); +} + +TEST(IntervalMapTest, EraseSplittingLeft) { + // Check that removal of a trailing subrange succeeds, but leaves the + // residual range in-place. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M; + + M.insert(7, 10, 42); + EXPECT_FALSE(M.empty()); + M.erase(9, 10); + EXPECT_EQ(std::next(M.begin()), M.end()); + EXPECT_EQ(M.begin()->first.first, 7U); + EXPECT_EQ(M.begin()->first.second, 9U); +} + +TEST(IntervalMapTest, EraseSplittingRight) { + // Check that removal of a leading subrange succeeds, but leaves the + // residual range in-place. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M; + + M.insert(7, 10, 42); + EXPECT_FALSE(M.empty()); + M.erase(7, 8); + EXPECT_EQ(std::next(M.begin()), M.end()); + EXPECT_EQ(M.begin()->first.first, 8U); + EXPECT_EQ(M.begin()->first.second, 10U); +} + +TEST(IntervalMapTest, EraseSplittingBoth) { + // Check that removal of an interior subrange leaves both the leading and + // trailing residual subranges in-place. + IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M; + + M.insert(7, 10, 42); + EXPECT_FALSE(M.empty()); + M.erase(8, 9); + EXPECT_EQ(std::next(std::next(M.begin())), M.end()); + EXPECT_EQ(M.begin()->first.first, 7U); + EXPECT_EQ(M.begin()->first.second, 8U); + EXPECT_EQ(std::next(M.begin())->first.first, 9U); + EXPECT_EQ(std::next(M.begin())->first.second, 10U); +} + +TEST(IntervalMapTest, NonCoalescingMapPermitsNonComparableKeys) { + // Test that values that can't be equality-compared are still usable when + // coalescing is disabled and behave as expected. + + struct S {}; // Struct with no equality comparison. + + IntervalMap<unsigned, S, IntervalCoalescing::Disabled> M; + + M.insert(7, 8, S()); + + EXPECT_FALSE(M.empty()); + EXPECT_EQ(std::next(M.begin()), M.end()); + EXPECT_EQ(M.find(7), M.begin()); + EXPECT_EQ(M.find(8), M.end()); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/interval_set_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/interval_set_test.cpp new file mode 100644 index 000000000000..7971a55f271f --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/interval_set_test.cpp @@ -0,0 +1,121 @@ +//===-- interval_set_test.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#include "interval_set.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +TEST(IntervalSetTest, DefaultConstructed) { + // Check that a default-constructed IntervalSet behaves as expected. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + EXPECT_TRUE(S.empty()); + EXPECT_TRUE(S.begin() == S.end()); + EXPECT_TRUE(S.find(0) == S.end()); +} + +TEST(IntervalSetTest, InsertSingleElement) { + // Check that a set with a single element inserted behaves as expected. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + S.insert(7, 8); + + EXPECT_FALSE(S.empty()); + EXPECT_EQ(std::next(S.begin()), S.end()); + EXPECT_EQ(S.find(7), S.begin()); + EXPECT_EQ(S.find(8), S.end()); +} + +TEST(IntervalSetTest, InsertCoalesceWithPrevious) { + // Check that insertions coalesce with previous ranges. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + S.insert(7, 8); + S.insert(8, 9); + + EXPECT_FALSE(S.empty()); + EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range. + EXPECT_EQ(S.find(7), S.find(8)); // 7 and 8 should point to same range. +} + +TEST(IntervalSetTest, InsertCoalesceWithFollowing) { + // Check that insertions coalesce with following ranges. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + S.insert(8, 9); + S.insert(7, 8); + + EXPECT_FALSE(S.empty()); + EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range. + EXPECT_EQ(S.find(7), S.find(8)); // 7 and 8 should point to same range. +} + +TEST(IntervalSetTest, InsertCoalesceBoth) { + // Check that insertions coalesce with ranges on both sides. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + S.insert(7, 8); + S.insert(9, 10); + + // Check no coalescing yet. + EXPECT_NE(S.find(7), S.find(9)); + + // Insert a 3rd range to trigger coalescing on both sides. + S.insert(8, 9); + + EXPECT_FALSE(S.empty()); + EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range. + EXPECT_EQ(S.find(7), S.find(8)); // 7, 8, and 9 should point to same range. + EXPECT_EQ(S.find(8), S.find(9)); +} + +TEST(IntervalSetTest, EraseSplittingLeft) { + // Check that removal of a trailing subrange succeeds, but leaves the + // residual range in-place. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + S.insert(7, 10); + EXPECT_FALSE(S.empty()); + S.erase(9, 10); + EXPECT_EQ(std::next(S.begin()), S.end()); + EXPECT_EQ(S.begin()->first, 7U); + EXPECT_EQ(S.begin()->second, 9U); +} + +TEST(IntervalSetTest, EraseSplittingRight) { + // Check that removal of a leading subrange succeeds, but leaves the + // residual range in-place. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + S.insert(7, 10); + EXPECT_FALSE(S.empty()); + S.erase(7, 8); + EXPECT_EQ(std::next(S.begin()), S.end()); + EXPECT_EQ(S.begin()->first, 8U); + EXPECT_EQ(S.begin()->second, 10U); +} + +TEST(IntervalSetTest, EraseSplittingBoth) { + // Check that removal of an interior subrange leaves both the leading and + // trailing residual subranges in-place. + IntervalSet<unsigned, IntervalCoalescing::Enabled> S; + + S.insert(7, 10); + EXPECT_FALSE(S.empty()); + S.erase(8, 9); + EXPECT_EQ(std::next(std::next(S.begin())), S.end()); + EXPECT_EQ(S.begin()->first, 7U); + EXPECT_EQ(S.begin()->second, 8U); + EXPECT_EQ(std::next(S.begin())->first, 9U); + EXPECT_EQ(std::next(S.begin())->second, 10U); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/orc_unit_test_main.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/orc_unit_test_main.cpp new file mode 100644 index 000000000000..d02101704d65 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/orc_unit_test_main.cpp @@ -0,0 +1,18 @@ +//===-- orc_unit_test_main.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// +#include "gtest/gtest.h" + +int main(int argc, char **argv) { + testing::GTEST_FLAG(death_test_style) = "threadsafe"; + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_test.cpp new file mode 100644 index 000000000000..397114b4017e --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_test.cpp @@ -0,0 +1,184 @@ +//===-- simple_packed_serialization_test.cpp ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#include "simple_packed_serialization.h" +#include "simple_packed_serialization_utils.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +TEST(SimplePackedSerializationTest, SPSOutputBuffer) { + constexpr unsigned NumBytes = 8; + char Buffer[NumBytes]; + char Zero = 0; + SPSOutputBuffer OB(Buffer, NumBytes); + + // Expect that we can write NumBytes of content. + for (unsigned I = 0; I != NumBytes; ++I) { + char C = I; + EXPECT_TRUE(OB.write(&C, 1)); + } + + // Expect an error when we attempt to write an extra byte. + EXPECT_FALSE(OB.write(&Zero, 1)); + + // Check that the buffer contains the expected content. + for (unsigned I = 0; I != NumBytes; ++I) + EXPECT_EQ(Buffer[I], (char)I); +} + +TEST(SimplePackedSerializationTest, SPSInputBuffer) { + char Buffer[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}; + SPSInputBuffer IB(Buffer, sizeof(Buffer)); + + char C; + for (unsigned I = 0; I != sizeof(Buffer); ++I) { + EXPECT_TRUE(IB.read(&C, 1)); + EXPECT_EQ(C, (char)I); + } + + EXPECT_FALSE(IB.read(&C, 1)); +} + +template <typename T> static void testFixedIntegralTypeSerialization() { + blobSerializationRoundTrip<T, T>(0); + blobSerializationRoundTrip<T, T>(static_cast<T>(1)); + if (std::is_signed<T>::value) { + blobSerializationRoundTrip<T, T>(static_cast<T>(-1)); + blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::min()); + } + blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::max()); +} + +TEST(SimplePackedSerializationTest, BoolSerialization) { + blobSerializationRoundTrip<bool, bool>(true); + blobSerializationRoundTrip<bool, bool>(false); +} + +TEST(SimplePackedSerializationTest, CharSerialization) { + blobSerializationRoundTrip<char, char>((char)0x00); + blobSerializationRoundTrip<char, char>((char)0xAA); + blobSerializationRoundTrip<char, char>((char)0xFF); +} + +TEST(SimplePackedSerializationTest, Int8Serialization) { + testFixedIntegralTypeSerialization<int8_t>(); +} + +TEST(SimplePackedSerializationTest, UInt8Serialization) { + testFixedIntegralTypeSerialization<uint8_t>(); +} + +TEST(SimplePackedSerializationTest, Int16Serialization) { + testFixedIntegralTypeSerialization<int16_t>(); +} + +TEST(SimplePackedSerializationTest, UInt16Serialization) { + testFixedIntegralTypeSerialization<uint16_t>(); +} + +TEST(SimplePackedSerializationTest, Int32Serialization) { + testFixedIntegralTypeSerialization<int32_t>(); +} + +TEST(SimplePackedSerializationTest, UInt32Serialization) { + testFixedIntegralTypeSerialization<uint32_t>(); +} + +TEST(SimplePackedSerializationTest, Int64Serialization) { + testFixedIntegralTypeSerialization<int64_t>(); +} + +TEST(SimplePackedSerializationTest, UInt64Serialization) { + testFixedIntegralTypeSerialization<uint64_t>(); +} + +TEST(SimplePackedSerializationTest, SequenceSerialization) { + std::vector<int32_t> V({1, 2, -47, 139}); + blobSerializationRoundTrip<SPSSequence<int32_t>, std::vector<int32_t>>(V); +} + +TEST(SimplePackedSerializationTest, StringViewCharSequenceSerialization) { + const char *HW = "Hello, world!"; + blobSerializationRoundTrip<SPSString, std::string_view>(std::string_view(HW)); +} + +TEST(SimplePackedSerializationTest, SpanSerialization) { + const char Data[] = {3, 2, 1, 0, 1, 2, 3}; // Span should handle nulls. + span<const char> OutS(Data, sizeof(Data)); + + size_t Size = SPSArgList<SPSSequence<char>>::size(OutS); + auto Buffer = std::make_unique<char[]>(Size); + SPSOutputBuffer OB(Buffer.get(), Size); + + EXPECT_TRUE(SPSArgList<SPSSequence<char>>::serialize(OB, OutS)); + + SPSInputBuffer IB(Buffer.get(), Size); + + span<const char> InS; + + EXPECT_TRUE(SPSArgList<SPSSequence<char>>::deserialize(IB, InS)); + + // Check that the serialized and deserialized values match. + EXPECT_EQ(InS.size(), OutS.size()); + EXPECT_EQ(memcmp(OutS.data(), InS.data(), InS.size()), 0); + + // Check that the span points directly to the input buffer. + EXPECT_EQ(InS.data(), Buffer.get() + sizeof(uint64_t)); +} + +TEST(SimplePackedSerializationTest, StdTupleSerialization) { + std::tuple<int32_t, std::string, bool> P(42, "foo", true); + blobSerializationRoundTrip<SPSTuple<int32_t, SPSString, bool>>(P); +} + +TEST(SimplePackedSerializationTest, StdPairSerialization) { + std::pair<int32_t, std::string> P(42, "foo"); + blobSerializationRoundTrip<SPSTuple<int32_t, SPSString>, + std::pair<int32_t, std::string>>(P); +} + +TEST(SimplePackedSerializationTest, StdOptionalNoValueSerialization) { + std::optional<int64_t> NoValue; + blobSerializationRoundTrip<SPSOptional<int64_t>>(NoValue); +} + +TEST(SimplePackedSerializationTest, StdOptionalValueSerialization) { + std::optional<int64_t> Value(42); + blobSerializationRoundTrip<SPSOptional<int64_t>>(Value); +} + +TEST(SimplePackedSerializationTest, ArgListSerialization) { + using BAL = SPSArgList<bool, int32_t, SPSString>; + + bool Arg1 = true; + int32_t Arg2 = 42; + std::string Arg3 = "foo"; + + size_t Size = BAL::size(Arg1, Arg2, Arg3); + auto Buffer = std::make_unique<char[]>(Size); + SPSOutputBuffer OB(Buffer.get(), Size); + + EXPECT_TRUE(BAL::serialize(OB, Arg1, Arg2, Arg3)); + + SPSInputBuffer IB(Buffer.get(), Size); + + bool ArgOut1; + int32_t ArgOut2; + std::string ArgOut3; + + EXPECT_TRUE(BAL::deserialize(IB, ArgOut1, ArgOut2, ArgOut3)); + + EXPECT_EQ(Arg1, ArgOut1); + EXPECT_EQ(Arg2, ArgOut2); + EXPECT_EQ(Arg3, ArgOut3); +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_utils.h b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_utils.h new file mode 100644 index 000000000000..746be43d250b --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_utils.h @@ -0,0 +1,34 @@ +//===-- simple_packed_serialization_utils.h -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_TEST_SIMPLE_PACKED_SERIALIZATION_UTILS_H +#define ORC_RT_TEST_SIMPLE_PACKED_SERIALIZATION_UTILS_H + +#include "simple_packed_serialization.h" +#include "gtest/gtest.h" + +template <typename SPSTagT, typename T> +static void blobSerializationRoundTrip(const T &Value) { + using BST = __orc_rt::SPSSerializationTraits<SPSTagT, T>; + + size_t Size = BST::size(Value); + auto Buffer = std::make_unique<char[]>(Size); + __orc_rt::SPSOutputBuffer OB(Buffer.get(), Size); + + EXPECT_TRUE(BST::serialize(OB, Value)); + + __orc_rt::SPSInputBuffer IB(Buffer.get(), Size); + + T DSValue; + EXPECT_TRUE(BST::deserialize(IB, DSValue)); + + EXPECT_EQ(Value, DSValue) + << "Incorrect value after serialization/deserialization round-trip"; +} + +#endif // ORC_RT_TEST_SIMPLE_PACKED_SERIALIZATION_UTILS_H
\ No newline at end of file diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/string_pool_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/string_pool_test.cpp new file mode 100644 index 000000000000..15ee2ce7d24d --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/string_pool_test.cpp @@ -0,0 +1,66 @@ +//===---------- string_pool_test.cpp - Unit tests for StringPool ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "string_pool.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +namespace { + +TEST(StringPool, UniquingAndComparisons) { + StringPool SP; + auto P1 = SP.intern("hello"); + + std::string S("hel"); + S += "lo"; + auto P2 = SP.intern(S); + + auto P3 = SP.intern("goodbye"); + + EXPECT_EQ(P1, P2) << "Failed to unique entries"; + EXPECT_NE(P1, P3) << "Unequal pooled strings comparing equal"; + + // We want to test that less-than comparison of PooledStringPtrs compiles, + // however we can't test the actual result as this is a pointer comparison and + // PooledStringPtr doesn't expose the underlying address of the string. + (void)(P1 < P3); +} + +TEST(StringPool, Dereference) { + StringPool SP; + auto Foo = SP.intern("foo"); + EXPECT_EQ(*Foo, "foo") << "Equality on dereferenced string failed"; +} + +TEST(StringPool, ClearDeadEntries) { + StringPool SP; + { + auto P1 = SP.intern("s1"); + SP.clearDeadEntries(); + EXPECT_FALSE(SP.empty()) << "\"s1\" entry in pool should still be retained"; + } + SP.clearDeadEntries(); + EXPECT_TRUE(SP.empty()) << "pool should be empty"; +} + +TEST(StringPool, NullPtr) { + // Make sure that we can default construct and then destroy a null + // PooledStringPtr. + PooledStringPtr Null; +} + +TEST(StringPool, Hashable) { + StringPool SP; + PooledStringPtr P1 = SP.intern("s1"); + PooledStringPtr Null; + EXPECT_NE(std::hash<PooledStringPtr>()(P1), + std::hash<PooledStringPtr>()(Null)); +} + +} // end anonymous namespace diff --git a/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/wrapper_function_utils_test.cpp b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/wrapper_function_utils_test.cpp new file mode 100644 index 000000000000..f10c5093046d --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/tests/unit/wrapper_function_utils_test.cpp @@ -0,0 +1,184 @@ +//===-- wrapper_function_utils_test.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#include "wrapper_function_utils.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +namespace { +constexpr const char *TestString = "test string"; +} // end anonymous namespace + +TEST(WrapperFunctionUtilsTest, DefaultWrapperFunctionResult) { + WrapperFunctionResult R; + EXPECT_TRUE(R.empty()); + EXPECT_EQ(R.size(), 0U); + EXPECT_EQ(R.getOutOfBandError(), nullptr); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCStruct) { + orc_rt_CWrapperFunctionResult CR = + orc_rt_CreateCWrapperFunctionResultFromString(TestString); + WrapperFunctionResult R(CR); + EXPECT_EQ(R.size(), strlen(TestString) + 1); + EXPECT_TRUE(strcmp(R.data(), TestString) == 0); + EXPECT_FALSE(R.empty()); + EXPECT_EQ(R.getOutOfBandError(), nullptr); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromRange) { + auto R = WrapperFunctionResult::copyFrom(TestString, strlen(TestString) + 1); + EXPECT_EQ(R.size(), strlen(TestString) + 1); + EXPECT_TRUE(strcmp(R.data(), TestString) == 0); + EXPECT_FALSE(R.empty()); + EXPECT_EQ(R.getOutOfBandError(), nullptr); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCString) { + auto R = WrapperFunctionResult::copyFrom(TestString); + EXPECT_EQ(R.size(), strlen(TestString) + 1); + EXPECT_TRUE(strcmp(R.data(), TestString) == 0); + EXPECT_FALSE(R.empty()); + EXPECT_EQ(R.getOutOfBandError(), nullptr); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromStdString) { + auto R = WrapperFunctionResult::copyFrom(std::string(TestString)); + EXPECT_EQ(R.size(), strlen(TestString) + 1); + EXPECT_TRUE(strcmp(R.data(), TestString) == 0); + EXPECT_FALSE(R.empty()); + EXPECT_EQ(R.getOutOfBandError(), nullptr); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromOutOfBandError) { + auto R = WrapperFunctionResult::createOutOfBandError(TestString); + EXPECT_FALSE(R.empty()); + EXPECT_TRUE(strcmp(R.getOutOfBandError(), TestString) == 0); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionCCallCreateEmpty) { + EXPECT_TRUE(!!WrapperFunctionCall::Create<SPSArgList<>>(ExecutorAddr())); +} + +static void voidNoop() {} + +static orc_rt_CWrapperFunctionResult voidNoopWrapper(const char *ArgData, + size_t ArgSize) { + return WrapperFunction<void()>::handle(ArgData, ArgSize, voidNoop).release(); +} + +static orc_rt_CWrapperFunctionResult addWrapper(const char *ArgData, + size_t ArgSize) { + return WrapperFunction<int32_t(int32_t, int32_t)>::handle( + ArgData, ArgSize, + [](int32_t X, int32_t Y) -> int32_t { return X + Y; }) + .release(); +} + +extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx{}; + +extern "C" orc_rt_CWrapperFunctionResult +__orc_rt_jit_dispatch(__orc_rt_Opaque *Ctx, const void *FnTag, + const char *ArgData, size_t ArgSize) { + using WrapperFunctionType = + orc_rt_CWrapperFunctionResult (*)(const char *, size_t); + + return reinterpret_cast<WrapperFunctionType>(const_cast<void *>(FnTag))( + ArgData, ArgSize); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionCallVoidNoopAndHandle) { + EXPECT_FALSE(!!WrapperFunction<void()>::call((void *)&voidNoopWrapper)); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionCallAddWrapperAndHandle) { + int32_t Result; + EXPECT_FALSE(!!WrapperFunction<int32_t(int32_t, int32_t)>::call( + (void *)&addWrapper, Result, 1, 2)); + EXPECT_EQ(Result, (int32_t)3); +} + +class AddClass { +public: + AddClass(int32_t X) : X(X) {} + int32_t addMethod(int32_t Y) { return X + Y; } + +private: + int32_t X; +}; + +static orc_rt_CWrapperFunctionResult addMethodWrapper(const char *ArgData, + size_t ArgSize) { + return WrapperFunction<int32_t(SPSExecutorAddr, int32_t)>::handle( + ArgData, ArgSize, makeMethodWrapperHandler(&AddClass::addMethod)) + .release(); +} + +TEST(WrapperFunctionUtilsTest, WrapperFunctionMethodCallAndHandleRet) { + int32_t Result; + AddClass AddObj(1); + EXPECT_FALSE(!!WrapperFunction<int32_t(SPSExecutorAddr, int32_t)>::call( + (void *)&addMethodWrapper, Result, ExecutorAddr::fromPtr(&AddObj), 2)); + EXPECT_EQ(Result, (int32_t)3); +} + +static orc_rt_CWrapperFunctionResult sumArrayWrapper(const char *ArgData, + size_t ArgSize) { + return WrapperFunction<int8_t(SPSExecutorAddrRange)>::handle( + ArgData, ArgSize, + [](ExecutorAddrRange R) { + int8_t Sum = 0; + for (char C : R.toSpan<char>()) + Sum += C; + return Sum; + }) + .release(); +} + +TEST(WrapperFunctionUtilsTest, SerializedWrapperFunctionCallTest) { + { + // Check wrapper function calls. + char A[] = {1, 2, 3, 4}; + + auto WFC = + cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( + ExecutorAddr::fromPtr(sumArrayWrapper), + ExecutorAddrRange(ExecutorAddr::fromPtr(A), + ExecutorAddrDiff(sizeof(A))))); + + WrapperFunctionResult WFR(WFC.run()); + EXPECT_EQ(WFR.size(), 1U); + EXPECT_EQ(WFR.data()[0], 10); + } + + { + // Check calls to void functions. + auto WFC = + cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( + ExecutorAddr::fromPtr(voidNoopWrapper), ExecutorAddrRange())); + auto Err = WFC.runWithSPSRet<void>(); + EXPECT_FALSE(!!Err); + } + + { + // Check calls with arguments and return values. + auto WFC = + cantFail(WrapperFunctionCall::Create<SPSArgList<int32_t, int32_t>>( + ExecutorAddr::fromPtr(addWrapper), 2, 4)); + + int32_t Result = 0; + auto Err = WFC.runWithSPSRet<int32_t>(Result); + EXPECT_FALSE(!!Err); + EXPECT_EQ(Result, 6); + } +} diff --git a/contrib/llvm-project/compiler-rt/lib/orc/wrapper_function_utils.h b/contrib/llvm-project/compiler-rt/lib/orc/wrapper_function_utils.h new file mode 100644 index 000000000000..8009438547a3 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/orc/wrapper_function_utils.h @@ -0,0 +1,509 @@ +//===-- wrapper_function_utils.h - Utilities for wrapper funcs --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_WRAPPER_FUNCTION_UTILS_H +#define ORC_RT_WRAPPER_FUNCTION_UTILS_H + +#include "orc_rt/c_api.h" +#include "common.h" +#include "error.h" +#include "executor_address.h" +#include "simple_packed_serialization.h" +#include <type_traits> + +namespace __orc_rt { + +/// C++ wrapper function result: Same as CWrapperFunctionResult but +/// auto-releases memory. +class WrapperFunctionResult { +public: + /// Create a default WrapperFunctionResult. + WrapperFunctionResult() { orc_rt_CWrapperFunctionResultInit(&R); } + + /// Create a WrapperFunctionResult from a CWrapperFunctionResult. This + /// instance takes ownership of the result object and will automatically + /// call dispose on the result upon destruction. + WrapperFunctionResult(orc_rt_CWrapperFunctionResult R) : R(R) {} + + WrapperFunctionResult(const WrapperFunctionResult &) = delete; + WrapperFunctionResult &operator=(const WrapperFunctionResult &) = delete; + + WrapperFunctionResult(WrapperFunctionResult &&Other) { + orc_rt_CWrapperFunctionResultInit(&R); + std::swap(R, Other.R); + } + + WrapperFunctionResult &operator=(WrapperFunctionResult &&Other) { + orc_rt_CWrapperFunctionResult Tmp; + orc_rt_CWrapperFunctionResultInit(&Tmp); + std::swap(Tmp, Other.R); + std::swap(R, Tmp); + return *this; + } + + ~WrapperFunctionResult() { orc_rt_DisposeCWrapperFunctionResult(&R); } + + /// Relinquish ownership of and return the + /// orc_rt_CWrapperFunctionResult. + orc_rt_CWrapperFunctionResult release() { + orc_rt_CWrapperFunctionResult Tmp; + orc_rt_CWrapperFunctionResultInit(&Tmp); + std::swap(R, Tmp); + return Tmp; + } + + /// Get a pointer to the data contained in this instance. + char *data() { return orc_rt_CWrapperFunctionResultData(&R); } + + /// Returns the size of the data contained in this instance. + size_t size() const { return orc_rt_CWrapperFunctionResultSize(&R); } + + /// Returns true if this value is equivalent to a default-constructed + /// WrapperFunctionResult. + bool empty() const { return orc_rt_CWrapperFunctionResultEmpty(&R); } + + /// Create a WrapperFunctionResult with the given size and return a pointer + /// to the underlying memory. + static WrapperFunctionResult allocate(size_t Size) { + WrapperFunctionResult R; + R.R = orc_rt_CWrapperFunctionResultAllocate(Size); + return R; + } + + /// Copy from the given char range. + static WrapperFunctionResult copyFrom(const char *Source, size_t Size) { + return orc_rt_CreateCWrapperFunctionResultFromRange(Source, Size); + } + + /// Copy from the given null-terminated string (includes the null-terminator). + static WrapperFunctionResult copyFrom(const char *Source) { + return orc_rt_CreateCWrapperFunctionResultFromString(Source); + } + + /// Copy from the given std::string (includes the null terminator). + static WrapperFunctionResult copyFrom(const std::string &Source) { + return copyFrom(Source.c_str()); + } + + /// Create an out-of-band error by copying the given string. + static WrapperFunctionResult createOutOfBandError(const char *Msg) { + return orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(Msg); + } + + /// Create an out-of-band error by copying the given string. + static WrapperFunctionResult createOutOfBandError(const std::string &Msg) { + return createOutOfBandError(Msg.c_str()); + } + + template <typename SPSArgListT, typename... ArgTs> + static WrapperFunctionResult fromSPSArgs(const ArgTs &...Args) { + auto Result = allocate(SPSArgListT::size(Args...)); + SPSOutputBuffer OB(Result.data(), Result.size()); + if (!SPSArgListT::serialize(OB, Args...)) + return createOutOfBandError( + "Error serializing arguments to blob in call"); + return Result; + } + + /// If this value is an out-of-band error then this returns the error message, + /// otherwise returns nullptr. + const char *getOutOfBandError() const { + return orc_rt_CWrapperFunctionResultGetOutOfBandError(&R); + } + +private: + orc_rt_CWrapperFunctionResult R; +}; + +namespace detail { + +template <typename RetT> class WrapperFunctionHandlerCaller { +public: + template <typename HandlerT, typename ArgTupleT, std::size_t... I> + static decltype(auto) call(HandlerT &&H, ArgTupleT &Args, + std::index_sequence<I...>) { + return std::forward<HandlerT>(H)(std::get<I>(Args)...); + } +}; + +template <> class WrapperFunctionHandlerCaller<void> { +public: + template <typename HandlerT, typename ArgTupleT, std::size_t... I> + static SPSEmpty call(HandlerT &&H, ArgTupleT &Args, + std::index_sequence<I...>) { + std::forward<HandlerT>(H)(std::get<I>(Args)...); + return SPSEmpty(); + } +}; + +template <typename WrapperFunctionImplT, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper + : public WrapperFunctionHandlerHelper< + decltype(&std::remove_reference_t<WrapperFunctionImplT>::operator()), + ResultSerializer, SPSTagTs...> {}; + +template <typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> { +public: + using ArgTuple = std::tuple<std::decay_t<ArgTs>...>; + using ArgIndices = std::make_index_sequence<std::tuple_size<ArgTuple>::value>; + + template <typename HandlerT> + static WrapperFunctionResult apply(HandlerT &&H, const char *ArgData, + size_t ArgSize) { + ArgTuple Args; + if (!deserialize(ArgData, ArgSize, Args, ArgIndices{})) + return WrapperFunctionResult::createOutOfBandError( + "Could not deserialize arguments for wrapper function call"); + + auto HandlerResult = WrapperFunctionHandlerCaller<RetT>::call( + std::forward<HandlerT>(H), Args, ArgIndices{}); + + return ResultSerializer<decltype(HandlerResult)>::serialize( + std::move(HandlerResult)); + } + +private: + template <std::size_t... I> + static bool deserialize(const char *ArgData, size_t ArgSize, ArgTuple &Args, + std::index_sequence<I...>) { + SPSInputBuffer IB(ArgData, ArgSize); + return SPSArgList<SPSTagTs...>::deserialize(IB, std::get<I>(Args)...); + } +}; + +// Map function pointers to function types. +template <typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (*)(ArgTs...), ResultSerializer, + SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map non-const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...), ResultSerializer, + SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...) const, + ResultSerializer, SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +template <typename SPSRetTagT, typename RetT> class ResultSerializer { +public: + static WrapperFunctionResult serialize(RetT Result) { + return WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSRetTagT>>(Result); + } +}; + +template <typename SPSRetTagT> class ResultSerializer<SPSRetTagT, Error> { +public: + static WrapperFunctionResult serialize(Error Err) { + return WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(Err))); + } +}; + +template <typename SPSRetTagT, typename T> +class ResultSerializer<SPSRetTagT, Expected<T>> { +public: + static WrapperFunctionResult serialize(Expected<T> E) { + return WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(E))); + } +}; + +template <typename SPSRetTagT, typename RetT> class ResultDeserializer { +public: + static void makeSafe(RetT &Result) {} + + static Error deserialize(RetT &Result, const char *ArgData, size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + if (!SPSArgList<SPSRetTagT>::deserialize(IB, Result)) + return make_error<StringError>( + "Error deserializing return value from blob in call"); + return Error::success(); + } +}; + +template <> class ResultDeserializer<SPSError, Error> { +public: + static void makeSafe(Error &Err) { cantFail(std::move(Err)); } + + static Error deserialize(Error &Err, const char *ArgData, size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + SPSSerializableError BSE; + if (!SPSArgList<SPSError>::deserialize(IB, BSE)) + return make_error<StringError>( + "Error deserializing return value from blob in call"); + Err = fromSPSSerializable(std::move(BSE)); + return Error::success(); + } +}; + +template <typename SPSTagT, typename T> +class ResultDeserializer<SPSExpected<SPSTagT>, Expected<T>> { +public: + static void makeSafe(Expected<T> &E) { cantFail(E.takeError()); } + + static Error deserialize(Expected<T> &E, const char *ArgData, + size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + SPSSerializableExpected<T> BSE; + if (!SPSArgList<SPSExpected<SPSTagT>>::deserialize(IB, BSE)) + return make_error<StringError>( + "Error deserializing return value from blob in call"); + E = fromSPSSerializable(std::move(BSE)); + return Error::success(); + } +}; + +} // end namespace detail + +template <typename SPSSignature> class WrapperFunction; + +template <typename SPSRetTagT, typename... SPSTagTs> +class WrapperFunction<SPSRetTagT(SPSTagTs...)> { +private: + template <typename RetT> + using ResultSerializer = detail::ResultSerializer<SPSRetTagT, RetT>; + +public: + template <typename RetT, typename... ArgTs> + static Error call(const void *FnTag, RetT &Result, const ArgTs &...Args) { + + // RetT might be an Error or Expected value. Set the checked flag now: + // we don't want the user to have to check the unused result if this + // operation fails. + detail::ResultDeserializer<SPSRetTagT, RetT>::makeSafe(Result); + + // Since the functions cannot be zero/unresolved on Windows, the following + // reference taking would always be non-zero, thus generating a compiler + // warning otherwise. +#if !defined(_WIN32) + if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch_ctx)) + return make_error<StringError>("__orc_rt_jit_dispatch_ctx not set"); + if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch)) + return make_error<StringError>("__orc_rt_jit_dispatch not set"); +#endif + auto ArgBuffer = + WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSTagTs...>>(Args...); + if (const char *ErrMsg = ArgBuffer.getOutOfBandError()) + return make_error<StringError>(ErrMsg); + + WrapperFunctionResult ResultBuffer = __orc_rt_jit_dispatch( + &__orc_rt_jit_dispatch_ctx, FnTag, ArgBuffer.data(), ArgBuffer.size()); + if (auto ErrMsg = ResultBuffer.getOutOfBandError()) + return make_error<StringError>(ErrMsg); + + return detail::ResultDeserializer<SPSRetTagT, RetT>::deserialize( + Result, ResultBuffer.data(), ResultBuffer.size()); + } + + template <typename HandlerT> + static WrapperFunctionResult handle(const char *ArgData, size_t ArgSize, + HandlerT &&Handler) { + using WFHH = + detail::WrapperFunctionHandlerHelper<std::remove_reference_t<HandlerT>, + ResultSerializer, SPSTagTs...>; + return WFHH::apply(std::forward<HandlerT>(Handler), ArgData, ArgSize); + } + +private: + template <typename T> static const T &makeSerializable(const T &Value) { + return Value; + } + + static detail::SPSSerializableError makeSerializable(Error Err) { + return detail::toSPSSerializable(std::move(Err)); + } + + template <typename T> + static detail::SPSSerializableExpected<T> makeSerializable(Expected<T> E) { + return detail::toSPSSerializable(std::move(E)); + } +}; + +template <typename... SPSTagTs> +class WrapperFunction<void(SPSTagTs...)> + : private WrapperFunction<SPSEmpty(SPSTagTs...)> { +public: + template <typename... ArgTs> + static Error call(const void *FnTag, const ArgTs &...Args) { + SPSEmpty BE; + return WrapperFunction<SPSEmpty(SPSTagTs...)>::call(FnTag, BE, Args...); + } + + using WrapperFunction<SPSEmpty(SPSTagTs...)>::handle; +}; + +/// A function object that takes an ExecutorAddr as its first argument, +/// casts that address to a ClassT*, then calls the given method on that +/// pointer passing in the remaining function arguments. This utility +/// removes some of the boilerplate from writing wrappers for method calls. +/// +/// @code{.cpp} +/// class MyClass { +/// public: +/// void myMethod(uint32_t, bool) { ... } +/// }; +/// +/// // SPS Method signature -- note MyClass object address as first argument. +/// using SPSMyMethodWrapperSignature = +/// SPSTuple<SPSExecutorAddr, uint32_t, bool>; +/// +/// WrapperFunctionResult +/// myMethodCallWrapper(const char *ArgData, size_t ArgSize) { +/// return WrapperFunction<SPSMyMethodWrapperSignature>::handle( +/// ArgData, ArgSize, makeMethodWrapperHandler(&MyClass::myMethod)); +/// } +/// @endcode +/// +template <typename RetT, typename ClassT, typename... ArgTs> +class MethodWrapperHandler { +public: + using MethodT = RetT (ClassT::*)(ArgTs...); + MethodWrapperHandler(MethodT M) : M(M) {} + RetT operator()(ExecutorAddr ObjAddr, ArgTs &...Args) { + return (ObjAddr.toPtr<ClassT *>()->*M)(std::forward<ArgTs>(Args)...); + } + +private: + MethodT M; +}; + +/// Create a MethodWrapperHandler object from the given method pointer. +template <typename RetT, typename ClassT, typename... ArgTs> +MethodWrapperHandler<RetT, ClassT, ArgTs...> +makeMethodWrapperHandler(RetT (ClassT::*Method)(ArgTs...)) { + return MethodWrapperHandler<RetT, ClassT, ArgTs...>(Method); +} + +/// Represents a call to a wrapper function. +class WrapperFunctionCall { +public: + // FIXME: Switch to a SmallVector<char, 24> once ORC runtime has a + // smallvector. + using ArgDataBufferType = std::vector<char>; + + /// Create a WrapperFunctionCall using the given SPS serializer to serialize + /// the arguments. + template <typename SPSSerializer, typename... ArgTs> + static Expected<WrapperFunctionCall> Create(ExecutorAddr FnAddr, + const ArgTs &...Args) { + ArgDataBufferType ArgData; + ArgData.resize(SPSSerializer::size(Args...)); + SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(), + ArgData.size()); + if (SPSSerializer::serialize(OB, Args...)) + return WrapperFunctionCall(FnAddr, std::move(ArgData)); + return make_error<StringError>("Cannot serialize arguments for " + "AllocActionCall"); + } + + WrapperFunctionCall() = default; + + /// Create a WrapperFunctionCall from a target function and arg buffer. + WrapperFunctionCall(ExecutorAddr FnAddr, ArgDataBufferType ArgData) + : FnAddr(FnAddr), ArgData(std::move(ArgData)) {} + + /// Returns the address to be called. + const ExecutorAddr &getCallee() const { return FnAddr; } + + /// Returns the argument data. + const ArgDataBufferType &getArgData() const { return ArgData; } + + /// WrapperFunctionCalls convert to true if the callee is non-null. + explicit operator bool() const { return !!FnAddr; } + + /// Run call returning raw WrapperFunctionResult. + WrapperFunctionResult run() const { + using FnTy = + orc_rt_CWrapperFunctionResult(const char *ArgData, size_t ArgSize); + return WrapperFunctionResult( + FnAddr.toPtr<FnTy *>()(ArgData.data(), ArgData.size())); + } + + /// Run call and deserialize result using SPS. + template <typename SPSRetT, typename RetT> + std::enable_if_t<!std::is_same<SPSRetT, void>::value, Error> + runWithSPSRet(RetT &RetVal) const { + auto WFR = run(); + if (const char *ErrMsg = WFR.getOutOfBandError()) + return make_error<StringError>(ErrMsg); + SPSInputBuffer IB(WFR.data(), WFR.size()); + if (!SPSSerializationTraits<SPSRetT, RetT>::deserialize(IB, RetVal)) + return make_error<StringError>("Could not deserialize result from " + "serialized wrapper function call"); + return Error::success(); + } + + /// Overload for SPS functions returning void. + template <typename SPSRetT> + std::enable_if_t<std::is_same<SPSRetT, void>::value, Error> + runWithSPSRet() const { + SPSEmpty E; + return runWithSPSRet<SPSEmpty>(E); + } + + /// Run call and deserialize an SPSError result. SPSError returns and + /// deserialization failures are merged into the returned error. + Error runWithSPSRetErrorMerged() const { + detail::SPSSerializableError RetErr; + if (auto Err = runWithSPSRet<SPSError>(RetErr)) + return Err; + return detail::fromSPSSerializable(std::move(RetErr)); + } + +private: + ExecutorAddr FnAddr; + std::vector<char> ArgData; +}; + +using SPSWrapperFunctionCall = SPSTuple<SPSExecutorAddr, SPSSequence<char>>; + +template <> +class SPSSerializationTraits<SPSWrapperFunctionCall, WrapperFunctionCall> { +public: + static size_t size(const WrapperFunctionCall &WFC) { + return SPSArgList<SPSExecutorAddr, SPSSequence<char>>::size( + WFC.getCallee(), WFC.getArgData()); + } + + static bool serialize(SPSOutputBuffer &OB, const WrapperFunctionCall &WFC) { + return SPSArgList<SPSExecutorAddr, SPSSequence<char>>::serialize( + OB, WFC.getCallee(), WFC.getArgData()); + } + + static bool deserialize(SPSInputBuffer &IB, WrapperFunctionCall &WFC) { + ExecutorAddr FnAddr; + WrapperFunctionCall::ArgDataBufferType ArgData; + if (!SPSWrapperFunctionCall::AsArgList::deserialize(IB, FnAddr, ArgData)) + return false; + WFC = WrapperFunctionCall(FnAddr, std::move(ArgData)); + return true; + } +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_WRAPPER_FUNCTION_UTILS_H |