diff options
Diffstat (limited to 'lib/msan/msan_allocator.cpp')
-rw-r--r-- | lib/msan/msan_allocator.cpp | 349 |
1 files changed, 349 insertions, 0 deletions
diff --git a/lib/msan/msan_allocator.cpp b/lib/msan/msan_allocator.cpp new file mode 100644 index 000000000000..6aa4e2738075 --- /dev/null +++ b/lib/msan/msan_allocator.cpp @@ -0,0 +1,349 @@ +//===-- msan_allocator.cpp -------------------------- ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemorySanitizer. +// +// MemorySanitizer allocator. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_report.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "msan.h" +#include "msan_allocator.h" +#include "msan_origin.h" +#include "msan_thread.h" +#include "msan_poisoning.h" + +namespace __msan { + +struct Metadata { + uptr requested_size; +}; + +struct MsanMapUnmapCallback { + void OnMap(uptr p, uptr size) const {} + void OnUnmap(uptr p, uptr size) const { + __msan_unpoison((void *)p, size); + + // We are about to unmap a chunk of user memory. + // Mark the corresponding shadow memory as not needed. + uptr shadow_p = MEM_TO_SHADOW(p); + ReleaseMemoryPagesToOS(shadow_p, shadow_p + size); + if (__msan_get_track_origins()) { + uptr origin_p = MEM_TO_ORIGIN(p); + ReleaseMemoryPagesToOS(origin_p, origin_p + size); + } + } +}; + +#if defined(__mips64) +static const uptr kMaxAllowedMallocSize = 2UL << 30; + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = 20; + using AddressSpaceView = LocalAddressSpaceView; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32<AP32> PrimaryAllocator; +#elif defined(__x86_64__) +#if SANITIZER_NETBSD || \ + (SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING)) +static const uptr kAllocatorSpace = 0x700000000000ULL; +#else +static const uptr kAllocatorSpace = 0x600000000000ULL; +#endif +static const uptr kMaxAllowedMallocSize = 8UL << 30; + +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = 0x40000000000; // 4T. + static const uptr kMetadataSize = sizeof(Metadata); + typedef DefaultSizeClassMap SizeClassMap; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; +}; + +typedef SizeClassAllocator64<AP64> PrimaryAllocator; + +#elif defined(__powerpc64__) +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = 0x300000000000; + static const uptr kSpaceSize = 0x020000000000; // 2T. + static const uptr kMetadataSize = sizeof(Metadata); + typedef DefaultSizeClassMap SizeClassMap; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; +}; + +typedef SizeClassAllocator64<AP64> PrimaryAllocator; +#elif defined(__aarch64__) +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = 20; + using AddressSpaceView = LocalAddressSpaceView; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32<AP32> PrimaryAllocator; +#endif +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; + +static Allocator allocator; +static AllocatorCache fallback_allocator_cache; +static StaticSpinMutex fallback_mutex; + +void MsanAllocatorInit() { + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + allocator.Init(common_flags()->allocator_release_to_os_interval_ms); +} + +AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) { + CHECK(ms); + CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); + return reinterpret_cast<AllocatorCache *>(ms->allocator_cache); +} + +void MsanThreadLocalMallocStorage::CommitBack() { + allocator.SwallowCache(GetAllocatorCache(this)); +} + +static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment, + bool zeroise) { + if (size > kMaxAllowedMallocSize) { + if (AllocatorMayReturnNull()) { + Report("WARNING: MemorySanitizer failed to allocate 0x%zx bytes\n", size); + return nullptr; + } + ReportAllocationSizeTooBig(size, kMaxAllowedMallocSize, stack); + } + MsanThread *t = GetCurrentThread(); + void *allocated; + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocated = allocator.Allocate(cache, size, alignment); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocated = allocator.Allocate(cache, size, alignment); + } + if (UNLIKELY(!allocated)) { + SetAllocatorOutOfMemory(); + if (AllocatorMayReturnNull()) + return nullptr; + ReportOutOfMemory(size, stack); + } + Metadata *meta = + reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated)); + meta->requested_size = size; + if (zeroise) { + __msan_clear_and_unpoison(allocated, size); + } else if (flags()->poison_in_malloc) { + __msan_poison(allocated, size); + if (__msan_get_track_origins()) { + stack->tag = StackTrace::TAG_ALLOC; + Origin o = Origin::CreateHeapOrigin(stack); + __msan_set_origin(allocated, size, o.raw_id()); + } + } + MSAN_MALLOC_HOOK(allocated, size); + return allocated; +} + +void MsanDeallocate(StackTrace *stack, void *p) { + CHECK(p); + MSAN_FREE_HOOK(p); + Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p)); + uptr size = meta->requested_size; + meta->requested_size = 0; + // This memory will not be reused by anyone else, so we are free to keep it + // poisoned. + if (flags()->poison_in_free) { + __msan_poison(p, size); + if (__msan_get_track_origins()) { + stack->tag = StackTrace::TAG_DEALLOC; + Origin o = Origin::CreateHeapOrigin(stack); + __msan_set_origin(p, size, o.raw_id()); + } + } + MsanThread *t = GetCurrentThread(); + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocator.Deallocate(cache, p); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocator.Deallocate(cache, p); + } +} + +void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, + uptr alignment) { + Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p)); + uptr old_size = meta->requested_size; + uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); + if (new_size <= actually_allocated_size) { + // We are not reallocating here. + meta->requested_size = new_size; + if (new_size > old_size) { + if (flags()->poison_in_malloc) { + stack->tag = StackTrace::TAG_ALLOC; + PoisonMemory((char *)old_p + old_size, new_size - old_size, stack); + } + } + return old_p; + } + uptr memcpy_size = Min(new_size, old_size); + void *new_p = MsanAllocate(stack, new_size, alignment, false /*zeroise*/); + if (new_p) { + CopyMemory(new_p, old_p, memcpy_size, stack); + MsanDeallocate(stack, old_p); + } + return new_p; +} + +void *MsanCalloc(StackTrace *stack, uptr nmemb, uptr size) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportCallocOverflow(nmemb, size, stack); + } + return MsanAllocate(stack, nmemb * size, sizeof(u64), true); +} + +static uptr AllocationSize(const void *p) { + if (!p) return 0; + const void *beg = allocator.GetBlockBegin(p); + if (beg != p) return 0; + Metadata *b = (Metadata *)allocator.GetMetaData(p); + return b->requested_size; +} + +void *msan_malloc(uptr size, StackTrace *stack) { + return SetErrnoOnNull(MsanAllocate(stack, size, sizeof(u64), false)); +} + +void *msan_calloc(uptr nmemb, uptr size, StackTrace *stack) { + return SetErrnoOnNull(MsanCalloc(stack, nmemb, size)); +} + +void *msan_realloc(void *ptr, uptr size, StackTrace *stack) { + if (!ptr) + return SetErrnoOnNull(MsanAllocate(stack, size, sizeof(u64), false)); + if (size == 0) { + MsanDeallocate(stack, ptr); + return nullptr; + } + return SetErrnoOnNull(MsanReallocate(stack, ptr, size, sizeof(u64))); +} + +void *msan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return msan_realloc(ptr, nmemb * size, stack); +} + +void *msan_valloc(uptr size, StackTrace *stack) { + return SetErrnoOnNull(MsanAllocate(stack, size, GetPageSizeCached(), false)); +} + +void *msan_pvalloc(uptr size, StackTrace *stack) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportPvallocOverflow(size, stack); + } + // pvalloc(0) should allocate one page. + size = size ? RoundUpTo(size, PageSize) : PageSize; + return SetErrnoOnNull(MsanAllocate(stack, size, PageSize, false)); +} + +void *msan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAlignedAllocAlignment(size, alignment, stack); + } + return SetErrnoOnNull(MsanAllocate(stack, size, alignment, false)); +} + +void *msan_memalign(uptr alignment, uptr size, StackTrace *stack) { + if (UNLIKELY(!IsPowerOfTwo(alignment))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAllocationAlignment(alignment, stack); + } + return SetErrnoOnNull(MsanAllocate(stack, size, alignment, false)); +} + +int msan_posix_memalign(void **memptr, uptr alignment, uptr size, + StackTrace *stack) { + if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { + if (AllocatorMayReturnNull()) + return errno_EINVAL; + ReportInvalidPosixMemalignAlignment(alignment, stack); + } + void *ptr = MsanAllocate(stack, size, alignment, false); + if (UNLIKELY(!ptr)) + // OOM error is already taken care of by MsanAllocate. + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, alignment)); + *memptr = ptr; + return 0; +} + +} // namespace __msan + +using namespace __msan; + +uptr __sanitizer_get_current_allocated_bytes() { + uptr stats[AllocatorStatCount]; + allocator.GetStats(stats); + return stats[AllocatorStatAllocated]; +} + +uptr __sanitizer_get_heap_size() { + uptr stats[AllocatorStatCount]; + allocator.GetStats(stats); + return stats[AllocatorStatMapped]; +} + +uptr __sanitizer_get_free_bytes() { return 1; } + +uptr __sanitizer_get_unmapped_bytes() { return 1; } + +uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } + +int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; } + +uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); } |