1 files changed, 273 insertions, 0 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp
new file mode 100644
index 000000000000..6690dd07d99b
--- /dev/null
+++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp
@@ -0,0 +1,273 @@
+//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the section-based memory manager used by the MCJIT
+// execution engine and RuntimeDyld
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/Config/config.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Process.h"
+
+namespace llvm {
+
+uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
+                                                   unsigned Alignment,
+                                                   unsigned SectionID,
+                                                   StringRef SectionName,
+                                                   bool IsReadOnly) {
+  if (IsReadOnly)
+    return allocateSection(SectionMemoryManager::AllocationPurpose::ROData,
+                           Size, Alignment);
+  return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size,
+                         Alignment);
+}
+
+uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
+                                                   unsigned Alignment,
+                                                   unsigned SectionID,
+                                                   StringRef SectionName) {
+  return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size,
+                         Alignment);
+}
+
+uint8_t *SectionMemoryManager::allocateSection(
+    SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size,
+    unsigned Alignment) {
+  if (!Alignment)
+    Alignment = 16;
+
+  assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");
+
+  uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1);
+  uintptr_t Addr = 0;
+
+  MemoryGroup &MemGroup = [&]() -> MemoryGroup & {
+    switch (Purpose) {
+    case AllocationPurpose::Code:
+      return CodeMem;
+    case AllocationPurpose::ROData:
+      return RODataMem;
+    case AllocationPurpose::RWData:
+      return RWDataMem;
+    }
+    llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");
+  }();
+
+  // Look in the list of free memory regions and use a block there if one
+  // is available.
+  for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
+    if (FreeMB.Free.allocatedSize() >= RequiredSize) {
+      Addr = (uintptr_t)FreeMB.Free.base();
+      uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize();
+      // Align the address.
+      Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
+
+      if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
+        // The part of the block we're giving out to the user is now pending
+        MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
+
+        // Remember this pending block, such that future allocations can just
+        // modify it rather than creating a new one
+        FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
+      } else {
+        sys::MemoryBlock &PendingMB =
+            MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
+        PendingMB = sys::MemoryBlock(PendingMB.base(),
+                                     Addr + Size - (uintptr_t)PendingMB.base());
+      }
+
+      // Remember how much free space is now left in this block
+      FreeMB.Free =
+          sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
+      return (uint8_t *)Addr;
+    }
+  }
+
+  // No pre-allocated free block was large enough. Allocate a new memory region.
+  // Note that all sections get allocated as read-write.  The permissions will
+  // be updated later based on memory group.
+  //
+  // FIXME: It would be useful to define a default allocation size (or add
+  // it as a constructor parameter) to minimize the number of allocations.
+  //
+  // FIXME: Initialize the Near member for each memory group to avoid
+  // interleaving.
+  std::error_code ec;
+  sys::MemoryBlock MB = MMapper.allocateMappedMemory(
+      Purpose, RequiredSize, &MemGroup.Near,
+      sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);
+  if (ec) {
+    // FIXME: Add error propagation to the interface.
+    return nullptr;
+  }
+
+  // Save this address as the basis for our next request
+  MemGroup.Near = MB;
+
+  // Copy the address to all the other groups, if they have not
+  // been initialized.
+  if (CodeMem.Near.base() == 0)
+    CodeMem.Near = MB;
+  if (RODataMem.Near.base() == 0)
+    RODataMem.Near = MB;
+  if (RWDataMem.Near.base() == 0)
+    RWDataMem.Near = MB;
+
+  // Remember that we allocated this memory
+  MemGroup.AllocatedMem.push_back(MB);
+  Addr = (uintptr_t)MB.base();
+  uintptr_t EndOfBlock = Addr + MB.allocatedSize();
+
+  // Align the address.
+  Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
+
+  // The part of the block we're giving out to the user is now pending
+  MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
+
+  // The allocateMappedMemory may allocate much more memory than we need. In
+  // this case, we store the unused memory as a free memory block.
+  unsigned FreeSize = EndOfBlock - Addr - Size;
+  if (FreeSize > 16) {
+    FreeMemBlock FreeMB;
+    FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize);
+    FreeMB.PendingPrefixIndex = (unsigned)-1;
+    MemGroup.FreeMem.push_back(FreeMB);
+  }
+
+  // Return aligned address
+  return (uint8_t *)Addr;
+}
+
+bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) {
+  // FIXME: Should in-progress permissions be reverted if an error occurs?
+  std::error_code ec;
+
+  // Make code memory executable.
+  ec = applyMemoryGroupPermissions(CodeMem,
+                                   sys::Memory::MF_READ | sys::Memory::MF_EXEC);
+  if (ec) {
+    if (ErrMsg) {
+      *ErrMsg = ec.message();
+    }
+    return true;
+  }
+
+  // Make read-only data memory read-only.
+  ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ);
+  if (ec) {
+    if (ErrMsg) {
+      *ErrMsg = ec.message();
+    }
+    return true;
+  }
+
+  // Read-write data memory already has the correct permissions
+
+  // Some platforms with separate data cache and instruction cache require
+  // explicit cache flush, otherwise JIT code manipulations (like resolved
+  // relocations) will get to the data cache but not to the instruction cache.
+  invalidateInstructionCache();
+
+  return false;
+}
+
+static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
+  static const size_t PageSize = sys::Process::getPageSizeEstimate();
+
+  size_t StartOverlap =
+      (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
+
+  size_t TrimmedSize = M.allocatedSize();
+  TrimmedSize -= StartOverlap;
+  TrimmedSize -= TrimmedSize % PageSize;
+
+  sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap),
+                           TrimmedSize);
+
+  assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
+  assert((Trimmed.allocatedSize() % PageSize) == 0);
+  assert(M.base() <= Trimmed.base() &&
+         Trimmed.allocatedSize() <= M.allocatedSize());
+
+  return Trimmed;
+}
+
+std::error_code
+SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
+                                                  unsigned Permissions) {
+  for (sys::MemoryBlock &MB : MemGroup.PendingMem)
+    if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions))
+      return EC;
+
+  MemGroup.PendingMem.clear();
+
+  // Now go through free blocks and trim any of them that don't span the entire
+  // page because one of the pending blocks may have overlapped it.
+  for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
+    FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
+    // We cleared the PendingMem list, so all these pointers are now invalid
+    FreeMB.PendingPrefixIndex = (unsigned)-1;
+  }
+
+  // Remove all blocks which are now empty
+  erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) {
+    return FreeMB.Free.allocatedSize() == 0;
+  });
+
+  return std::error_code();
+}
+
+void SectionMemoryManager::invalidateInstructionCache() {
+  for (sys::MemoryBlock &Block : CodeMem.PendingMem)
+    sys::Memory::InvalidateInstructionCache(Block.base(),
+                                            Block.allocatedSize());
+}
+
+SectionMemoryManager::~SectionMemoryManager() {
+  for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
+    for (sys::MemoryBlock &Block : Group->AllocatedMem)
+      MMapper.releaseMappedMemory(Block);
+  }
+}
+
+SectionMemoryManager::MemoryMapper::~MemoryMapper() {}
+
+void SectionMemoryManager::anchor() {}
+
+namespace {
+// Trivial implementation of SectionMemoryManager::MemoryMapper that just calls
+// into sys::Memory.
+class DefaultMMapper final : public SectionMemoryManager::MemoryMapper {
+public:
+  sys::MemoryBlock
+  allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,
+                       size_t NumBytes, const sys::MemoryBlock *const NearBlock,
+                       unsigned Flags, std::error_code &EC) override {
+    return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);
+  }
+
+  std::error_code protectMappedMemory(const sys::MemoryBlock &Block,
+                                      unsigned Flags) override {
+    return sys::Memory::protectMappedMemory(Block, Flags);
+  }
+
+  std::error_code releaseMappedMemory(sys::MemoryBlock &M) override {
+    return sys::Memory::releaseMappedMemory(M);
+  }
+};
+
+DefaultMMapper DefaultMMapperInstance;
+} // namespace
+
+SectionMemoryManager::SectionMemoryManager(MemoryMapper *MM)
+    : MMapper(MM ? *MM : DefaultMMapperInstance) {}
+
+} // namespace llvm