diff options
Diffstat (limited to 'contrib/llvm-project/compiler-rt/lib/memprof')
37 files changed, 4176 insertions, 0 deletions
diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/README.txt b/contrib/llvm-project/compiler-rt/lib/memprof/README.txt new file mode 100644 index 000000000000..82012c5e71b0 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/README.txt @@ -0,0 +1,17 @@ +MemProfiling RT +================================ +This directory contains sources of the MemProfiling (MemProf) runtime library. + +Directory structure: +README.txt : This file. +CMakeLists.txt : File for cmake-based build. +memprof_*.{cc,h} : Sources of the memprof runtime library. + +Also MemProf runtime needs the following libraries: +lib/interception/ : Machinery used to intercept function calls. +lib/sanitizer_common/ : Code shared between various sanitizers. + +MemProf runtime can only be built by CMake. You can run MemProf tests +from the root of your CMake build tree: + +make check-memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof.syms.extra b/contrib/llvm-project/compiler-rt/lib/memprof/memprof.syms.extra new file mode 100644 index 000000000000..173280ffe97f --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof.syms.extra @@ -0,0 +1 @@ +__memprof_* diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_allocator.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_allocator.cpp new file mode 100644 index 000000000000..19b2b9010682 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_allocator.cpp @@ -0,0 +1,812 @@ +//===-- memprof_allocator.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Implementation of MemProf's memory allocator, which uses the allocator +// from sanitizer_common. +// +//===----------------------------------------------------------------------===// + +#include "memprof_allocator.h" +#include "memprof_mapping.h" +#include "memprof_mibmap.h" +#include "memprof_rawprofile.h" +#include "memprof_stack.h" +#include "memprof_thread.h" +#include "profile/MemProfData.inc" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_report.h" +#include "sanitizer_common/sanitizer_array_ref.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_file.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + +#include <sched.h> +#include <time.h> + +#define MAX_HISTOGRAM_PRINT_SIZE 32U + +extern bool __memprof_histogram; + +namespace __memprof { +namespace { +using ::llvm::memprof::MemInfoBlock; + +void Print(const MemInfoBlock &M, const u64 id, bool print_terse) { + u64 p; + + if (print_terse) { + p = M.TotalSize * 100 / M.AllocCount; + Printf("MIB:%llu/%u/%llu.%02llu/%u/%u/", id, M.AllocCount, p / 100, p % 100, + M.MinSize, M.MaxSize); + p = M.TotalAccessCount * 100 / M.AllocCount; + Printf("%llu.%02llu/%llu/%llu/", p / 100, p % 100, M.MinAccessCount, + M.MaxAccessCount); + p = M.TotalLifetime * 100 / M.AllocCount; + Printf("%llu.%02llu/%u/%u/", p / 100, p % 100, M.MinLifetime, + M.MaxLifetime); + Printf("%u/%u/%u/%u\n", M.NumMigratedCpu, M.NumLifetimeOverlaps, + M.NumSameAllocCpu, M.NumSameDeallocCpu); + } else { + p = M.TotalSize * 100 / M.AllocCount; + Printf("Memory allocation stack id = %llu\n", id); + Printf("\talloc_count %u, size (ave/min/max) %llu.%02llu / %u / %u\n", + M.AllocCount, p / 100, p % 100, M.MinSize, M.MaxSize); + p = M.TotalAccessCount * 100 / M.AllocCount; + Printf("\taccess_count (ave/min/max): %llu.%02llu / %llu / %llu\n", p / 100, + p % 100, M.MinAccessCount, M.MaxAccessCount); + p = M.TotalLifetime * 100 / M.AllocCount; + Printf("\tlifetime (ave/min/max): %llu.%02llu / %u / %u\n", p / 100, + p % 100, M.MinLifetime, M.MaxLifetime); + Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc " + "cpu: %u, num same dealloc_cpu: %u\n", + M.NumMigratedCpu, M.NumLifetimeOverlaps, M.NumSameAllocCpu, + M.NumSameDeallocCpu); + Printf("AccessCountHistogram[%u]: ", M.AccessHistogramSize); + uint32_t PrintSize = M.AccessHistogramSize > MAX_HISTOGRAM_PRINT_SIZE + ? MAX_HISTOGRAM_PRINT_SIZE + : M.AccessHistogramSize; + for (size_t i = 0; i < PrintSize; ++i) { + Printf("%llu ", ((uint64_t *)M.AccessHistogram)[i]); + } + Printf("\n"); + } +} +} // namespace + +static int GetCpuId(void) { + // _memprof_preinit is called via the preinit_array, which subsequently calls + // malloc. Since this is before _dl_init calls VDSO_SETUP, sched_getcpu + // will seg fault as the address of __vdso_getcpu will be null. + if (!memprof_inited) + return -1; + return sched_getcpu(); +} + +// Compute the timestamp in ms. +static int GetTimestamp(void) { + // timespec_get will segfault if called from dl_init + if (!memprof_timestamp_inited) { + // By returning 0, this will be effectively treated as being + // timestamped at memprof init time (when memprof_init_timestamp_s + // is initialized). + return 0; + } + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return (ts.tv_sec - memprof_init_timestamp_s) * 1000 + ts.tv_nsec / 1000000; +} + +static MemprofAllocator &get_allocator(); + +// The memory chunk allocated from the underlying allocator looks like this: +// H H U U U U U U +// H -- ChunkHeader (32 bytes) +// U -- user memory. + +// If there is left padding before the ChunkHeader (due to use of memalign), +// we store a magic value in the first uptr word of the memory block and +// store the address of ChunkHeader in the next uptr. +// M B L L L L L L L L L H H U U U U U U +// | ^ +// ---------------------| +// M -- magic value kAllocBegMagic +// B -- address of ChunkHeader pointing to the first 'H' + +constexpr uptr kMaxAllowedMallocBits = 40; + +// Should be no more than 32-bytes +struct ChunkHeader { + // 1-st 4 bytes. + u32 alloc_context_id; + // 2-nd 4 bytes + u32 cpu_id; + // 3-rd 4 bytes + u32 timestamp_ms; + // 4-th 4 bytes + // Note only 1 bit is needed for this flag if we need space in the future for + // more fields. + u32 from_memalign; + // 5-th and 6-th 4 bytes + // The max size of an allocation is 2^40 (kMaxAllowedMallocSize), so this + // could be shrunk to kMaxAllowedMallocBits if we need space in the future for + // more fields. + atomic_uint64_t user_requested_size; + // 23 bits available + // 7-th and 8-th 4 bytes + u64 data_type_id; // TODO: hash of type name +}; + +static const uptr kChunkHeaderSize = sizeof(ChunkHeader); +COMPILER_CHECK(kChunkHeaderSize == 32); + +struct MemprofChunk : ChunkHeader { + uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; } + uptr UsedSize() { + return atomic_load(&user_requested_size, memory_order_relaxed); + } + void *AllocBeg() { + if (from_memalign) + return get_allocator().GetBlockBegin(reinterpret_cast<void *>(this)); + return reinterpret_cast<void *>(this); + } +}; + +class LargeChunkHeader { + static constexpr uptr kAllocBegMagic = + FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL); + atomic_uintptr_t magic; + MemprofChunk *chunk_header; + +public: + MemprofChunk *Get() const { + return atomic_load(&magic, memory_order_acquire) == kAllocBegMagic + ? chunk_header + : nullptr; + } + + void Set(MemprofChunk *p) { + if (p) { + chunk_header = p; + atomic_store(&magic, kAllocBegMagic, memory_order_release); + return; + } + + uptr old = kAllocBegMagic; + if (!atomic_compare_exchange_strong(&magic, &old, 0, + memory_order_release)) { + CHECK_EQ(old, kAllocBegMagic); + } + } +}; + +void FlushUnneededMemProfShadowMemory(uptr p, uptr size) { + // Since memprof's mapping is compacting, the shadow chunk may be + // not page-aligned, so we only flush the page-aligned portion. + ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size)); +} + +void MemprofMapUnmapCallback::OnMap(uptr p, uptr size) const { + // Statistics. + MemprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mmaps++; + thread_stats.mmaped += size; +} + +void MemprofMapUnmapCallback::OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // Mark the corresponding shadow memory as not needed. + FlushUnneededMemProfShadowMemory(p, size); + // Statistics. + MemprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.munmaps++; + thread_stats.munmaped += size; +} + +AllocatorCache *GetAllocatorCache(MemprofThreadLocalMallocStorage *ms) { + CHECK(ms); + return &ms->allocator_cache; +} + +// Accumulates the access count from the shadow for the given pointer and size. +u64 GetShadowCount(uptr p, u32 size) { + u64 *shadow = (u64 *)MEM_TO_SHADOW(p); + u64 *shadow_end = (u64 *)MEM_TO_SHADOW(p + size); + u64 count = 0; + for (; shadow <= shadow_end; shadow++) + count += *shadow; + return count; +} + +// Accumulates the access count from the shadow for the given pointer and size. +// See memprof_mapping.h for an overview on histogram counters. +u64 GetShadowCountHistogram(uptr p, u32 size) { + u8 *shadow = (u8 *)HISTOGRAM_MEM_TO_SHADOW(p); + u8 *shadow_end = (u8 *)HISTOGRAM_MEM_TO_SHADOW(p + size); + u64 count = 0; + for (; shadow <= shadow_end; shadow++) + count += *shadow; + return count; +} + +// Clears the shadow counters (when memory is allocated). +void ClearShadow(uptr addr, uptr size) { + CHECK(AddrIsAlignedByGranularity(addr)); + CHECK(AddrIsInMem(addr)); + CHECK(AddrIsAlignedByGranularity(addr + size)); + CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY)); + CHECK(REAL(memset)); + uptr shadow_beg; + uptr shadow_end; + if (__memprof_histogram) { + shadow_beg = HISTOGRAM_MEM_TO_SHADOW(addr); + shadow_end = HISTOGRAM_MEM_TO_SHADOW(addr + size); + } else { + shadow_beg = MEM_TO_SHADOW(addr); + shadow_end = MEM_TO_SHADOW(addr + size - SHADOW_GRANULARITY) + 1; + } + + if (shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { + REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); + } else { + uptr page_size = GetPageSizeCached(); + uptr page_beg = RoundUpTo(shadow_beg, page_size); + uptr page_end = RoundDownTo(shadow_end, page_size); + + if (page_beg >= page_end) { + REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); + } else { + if (page_beg != shadow_beg) { + REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg); + } + if (page_end != shadow_end) { + REAL(memset)((void *)page_end, 0, shadow_end - page_end); + } + ReserveShadowMemoryRange(page_beg, page_end - 1, nullptr); + } + } +} + +struct Allocator { + static const uptr kMaxAllowedMallocSize = 1ULL << kMaxAllowedMallocBits; + + MemprofAllocator allocator; + StaticSpinMutex fallback_mutex; + AllocatorCache fallback_allocator_cache; + + uptr max_user_defined_malloc_size; + + // Holds the mapping of stack ids to MemInfoBlocks. + MIBMapTy MIBMap; + + atomic_uint8_t destructing; + atomic_uint8_t constructed; + bool print_text; + + // ------------------- Initialization ------------------------ + explicit Allocator(LinkerInitialized) : print_text(flags()->print_text) { + atomic_store_relaxed(&destructing, 0); + atomic_store_relaxed(&constructed, 1); + } + + ~Allocator() { + atomic_store_relaxed(&destructing, 1); + FinishAndWrite(); + } + + static void PrintCallback(const uptr Key, LockedMemInfoBlock *const &Value, + void *Arg) { + SpinMutexLock l(&Value->mutex); + Print(Value->mib, Key, bool(Arg)); + } + + // See memprof_mapping.h for an overview on histogram counters. + static MemInfoBlock CreateNewMIB(uptr p, MemprofChunk *m, u64 user_size) { + if (__memprof_histogram) { + return CreateNewMIBWithHistogram(p, m, user_size); + } else { + return CreateNewMIBWithoutHistogram(p, m, user_size); + } + } + + static MemInfoBlock CreateNewMIBWithHistogram(uptr p, MemprofChunk *m, + u64 user_size) { + + u64 c = GetShadowCountHistogram(p, user_size); + long curtime = GetTimestamp(); + uint32_t HistogramSize = + RoundUpTo(user_size, HISTOGRAM_GRANULARITY) / HISTOGRAM_GRANULARITY; + uintptr_t Histogram = + (uintptr_t)InternalAlloc(HistogramSize * sizeof(uint64_t)); + memset((void *)Histogram, 0, HistogramSize * sizeof(uint64_t)); + for (size_t i = 0; i < HistogramSize; ++i) { + u8 Counter = + *((u8 *)HISTOGRAM_MEM_TO_SHADOW(p + HISTOGRAM_GRANULARITY * i)); + ((uint64_t *)Histogram)[i] = (uint64_t)Counter; + } + MemInfoBlock newMIB(user_size, c, m->timestamp_ms, curtime, m->cpu_id, + GetCpuId(), Histogram, HistogramSize); + return newMIB; + } + + static MemInfoBlock CreateNewMIBWithoutHistogram(uptr p, MemprofChunk *m, + u64 user_size) { + u64 c = GetShadowCount(p, user_size); + long curtime = GetTimestamp(); + MemInfoBlock newMIB(user_size, c, m->timestamp_ms, curtime, m->cpu_id, + GetCpuId(), 0, 0); + return newMIB; + } + + void FinishAndWrite() { + if (print_text && common_flags()->print_module_map) + DumpProcessMap(); + + allocator.ForceLock(); + + InsertLiveBlocks(); + if (print_text) { + if (!flags()->print_terse) + Printf("Recorded MIBs (incl. live on exit):\n"); + MIBMap.ForEach(PrintCallback, + reinterpret_cast<void *>(flags()->print_terse)); + StackDepotPrintAll(); + } else { + // Serialize the contents to a raw profile. Format documented in + // memprof_rawprofile.h. + char *Buffer = nullptr; + + __sanitizer::ListOfModules List; + List.init(); + ArrayRef<LoadedModule> Modules(List.begin(), List.end()); + u64 BytesSerialized = SerializeToRawProfile(MIBMap, Modules, Buffer); + CHECK(Buffer && BytesSerialized && "could not serialize to buffer"); + report_file.Write(Buffer, BytesSerialized); + } + + allocator.ForceUnlock(); + } + + // Inserts any blocks which have been allocated but not yet deallocated. + void InsertLiveBlocks() { + allocator.ForEachChunk( + [](uptr chunk, void *alloc) { + u64 user_requested_size; + Allocator *A = (Allocator *)alloc; + MemprofChunk *m = + A->GetMemprofChunk((void *)chunk, user_requested_size); + if (!m) + return; + uptr user_beg = ((uptr)m) + kChunkHeaderSize; + MemInfoBlock newMIB = CreateNewMIB(user_beg, m, user_requested_size); + InsertOrMerge(m->alloc_context_id, newMIB, A->MIBMap); + }, + this); + } + + void InitLinkerInitialized() { + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + allocator.InitLinkerInitialized( + common_flags()->allocator_release_to_os_interval_ms); + max_user_defined_malloc_size = common_flags()->max_allocation_size_mb + ? common_flags()->max_allocation_size_mb + << 20 + : kMaxAllowedMallocSize; + } + + // -------------------- Allocation/Deallocation routines --------------- + void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, + AllocType alloc_type) { + if (UNLIKELY(!memprof_inited)) + MemprofInitFromRtl(); + if (UNLIKELY(IsRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportRssLimitExceeded(stack); + } + CHECK(stack); + const uptr min_alignment = MEMPROF_ALIGNMENT; + if (alignment < min_alignment) + alignment = min_alignment; + if (size == 0) { + // We'd be happy to avoid allocating memory for zero-size requests, but + // some programs/tests depend on this behavior and assume that malloc + // would not return NULL even for zero-size allocations. Moreover, it + // looks like operator new should never return NULL, and results of + // consecutive "new" calls must be different even if the allocated size + // is zero. + size = 1; + } + CHECK(IsPowerOfTwo(alignment)); + uptr rounded_size = RoundUpTo(size, alignment); + uptr needed_size = rounded_size + kChunkHeaderSize; + if (alignment > min_alignment) + needed_size += alignment; + CHECK(IsAligned(needed_size, min_alignment)); + if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize || + size > max_user_defined_malloc_size) { + if (AllocatorMayReturnNull()) { + Report("WARNING: MemProfiler failed to allocate 0x%zx bytes\n", size); + return nullptr; + } + uptr malloc_limit = + Min(kMaxAllowedMallocSize, max_user_defined_malloc_size); + ReportAllocationSizeTooBig(size, malloc_limit, stack); + } + + MemprofThread *t = GetCurrentThread(); + void *allocated; + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocated = allocator.Allocate(cache, needed_size, 8); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocated = allocator.Allocate(cache, needed_size, 8); + } + if (UNLIKELY(!allocated)) { + SetAllocatorOutOfMemory(); + if (AllocatorMayReturnNull()) + return nullptr; + ReportOutOfMemory(size, stack); + } + + uptr alloc_beg = reinterpret_cast<uptr>(allocated); + uptr alloc_end = alloc_beg + needed_size; + uptr beg_plus_header = alloc_beg + kChunkHeaderSize; + uptr user_beg = beg_plus_header; + if (!IsAligned(user_beg, alignment)) + user_beg = RoundUpTo(user_beg, alignment); + uptr user_end = user_beg + size; + CHECK_LE(user_end, alloc_end); + uptr chunk_beg = user_beg - kChunkHeaderSize; + MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg); + m->from_memalign = alloc_beg != chunk_beg; + CHECK(size); + + m->cpu_id = GetCpuId(); + m->timestamp_ms = GetTimestamp(); + m->alloc_context_id = StackDepotPut(*stack); + + uptr size_rounded_down_to_granularity = + RoundDownTo(size, SHADOW_GRANULARITY); + if (size_rounded_down_to_granularity) + ClearShadow(user_beg, size_rounded_down_to_granularity); + + MemprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mallocs++; + thread_stats.malloced += size; + thread_stats.malloced_overhead += needed_size - size; + if (needed_size > SizeClassMap::kMaxSize) + thread_stats.malloc_large++; + else + thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++; + + void *res = reinterpret_cast<void *>(user_beg); + atomic_store(&m->user_requested_size, size, memory_order_release); + if (alloc_beg != chunk_beg) { + CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg); + reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m); + } + RunMallocHooks(res, size); + return res; + } + + void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment, + BufferedStackTrace *stack, AllocType alloc_type) { + uptr p = reinterpret_cast<uptr>(ptr); + if (p == 0) + return; + + RunFreeHooks(ptr); + + uptr chunk_beg = p - kChunkHeaderSize; + MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg); + + u64 user_requested_size = + atomic_exchange(&m->user_requested_size, 0, memory_order_acquire); + if (memprof_inited && atomic_load_relaxed(&constructed) && + !atomic_load_relaxed(&destructing)) { + MemInfoBlock newMIB = this->CreateNewMIB(p, m, user_requested_size); + InsertOrMerge(m->alloc_context_id, newMIB, MIBMap); + } + + MemprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.frees++; + thread_stats.freed += user_requested_size; + + void *alloc_beg = m->AllocBeg(); + if (alloc_beg != m) { + // Clear the magic value, as allocator internals may overwrite the + // contents of deallocated chunk, confusing GetMemprofChunk lookup. + reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(nullptr); + } + + MemprofThread *t = GetCurrentThread(); + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocator.Deallocate(cache, alloc_beg); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocator.Deallocate(cache, alloc_beg); + } + } + + void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) { + CHECK(old_ptr && new_size); + uptr p = reinterpret_cast<uptr>(old_ptr); + uptr chunk_beg = p - kChunkHeaderSize; + MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg); + + MemprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.reallocs++; + thread_stats.realloced += new_size; + + void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC); + if (new_ptr) { + CHECK_NE(REAL(memcpy), nullptr); + uptr memcpy_size = Min(new_size, m->UsedSize()); + REAL(memcpy)(new_ptr, old_ptr, memcpy_size); + Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC); + } + return new_ptr; + } + + void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportCallocOverflow(nmemb, size, stack); + } + void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC); + // If the memory comes from the secondary allocator no need to clear it + // as it comes directly from mmap. + if (ptr && allocator.FromPrimary(ptr)) + REAL(memset)(ptr, 0, nmemb * size); + return ptr; + } + + void CommitBack(MemprofThreadLocalMallocStorage *ms) { + AllocatorCache *ac = GetAllocatorCache(ms); + allocator.SwallowCache(ac); + } + + // -------------------------- Chunk lookup ---------------------- + + // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). + MemprofChunk *GetMemprofChunk(void *alloc_beg, u64 &user_requested_size) { + if (!alloc_beg) + return nullptr; + MemprofChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get(); + if (!p) { + if (!allocator.FromPrimary(alloc_beg)) + return nullptr; + p = reinterpret_cast<MemprofChunk *>(alloc_beg); + } + // The size is reset to 0 on deallocation (and a min of 1 on + // allocation). + user_requested_size = + atomic_load(&p->user_requested_size, memory_order_acquire); + if (user_requested_size) + return p; + return nullptr; + } + + MemprofChunk *GetMemprofChunkByAddr(uptr p, u64 &user_requested_size) { + void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast<void *>(p)); + return GetMemprofChunk(alloc_beg, user_requested_size); + } + + uptr AllocationSize(uptr p) { + u64 user_requested_size; + MemprofChunk *m = GetMemprofChunkByAddr(p, user_requested_size); + if (!m) + return 0; + if (m->Beg() != p) + return 0; + return user_requested_size; + } + + uptr AllocationSizeFast(uptr p) { + return reinterpret_cast<MemprofChunk *>(p - kChunkHeaderSize)->UsedSize(); + } + + void Purge() { allocator.ForceReleaseToOS(); } + + void PrintStats() { allocator.PrintStats(); } + + void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + allocator.ForceLock(); + fallback_mutex.Lock(); + } + + void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + fallback_mutex.Unlock(); + allocator.ForceUnlock(); + } +}; + +static Allocator instance(LINKER_INITIALIZED); + +static MemprofAllocator &get_allocator() { return instance.allocator; } + +void InitializeAllocator() { instance.InitLinkerInitialized(); } + +void MemprofThreadLocalMallocStorage::CommitBack() { + instance.CommitBack(this); +} + +void PrintInternalAllocatorStats() { instance.PrintStats(); } + +void memprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { + instance.Deallocate(ptr, 0, 0, stack, alloc_type); +} + +void memprof_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type) { + instance.Deallocate(ptr, size, alignment, stack, alloc_type); +} + +void *memprof_malloc(uptr size, BufferedStackTrace *stack) { + return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC)); +} + +void *memprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { + return SetErrnoOnNull(instance.Calloc(nmemb, size, stack)); +} + +void *memprof_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return memprof_realloc(p, nmemb * size, stack); +} + +void *memprof_realloc(void *p, uptr size, BufferedStackTrace *stack) { + if (!p) + return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC)); + if (size == 0) { + if (flags()->allocator_frees_and_returns_null_on_realloc_zero) { + instance.Deallocate(p, 0, 0, stack, FROM_MALLOC); + return nullptr; + } + // Allocate a size of 1 if we shouldn't free() on Realloc to 0 + size = 1; + } + return SetErrnoOnNull(instance.Reallocate(p, size, stack)); +} + +void *memprof_valloc(uptr size, BufferedStackTrace *stack) { + return SetErrnoOnNull( + instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC)); +} + +void *memprof_pvalloc(uptr size, BufferedStackTrace *stack) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportPvallocOverflow(size, stack); + } + // pvalloc(0) should allocate one page. + size = size ? RoundUpTo(size, PageSize) : PageSize; + return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC)); +} + +void *memprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, + AllocType alloc_type) { + if (UNLIKELY(!IsPowerOfTwo(alignment))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAllocationAlignment(alignment, stack); + } + return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type)); +} + +void *memprof_aligned_alloc(uptr alignment, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAlignedAllocAlignment(size, alignment, stack); + } + return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC)); +} + +int memprof_posix_memalign(void **memptr, uptr alignment, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { + if (AllocatorMayReturnNull()) + return errno_EINVAL; + ReportInvalidPosixMemalignAlignment(alignment, stack); + } + void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC); + if (UNLIKELY(!ptr)) + // OOM error is already taken care of by Allocate. + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, alignment)); + *memptr = ptr; + return 0; +} + +static const void *memprof_malloc_begin(const void *p) { + u64 user_requested_size; + MemprofChunk *m = + instance.GetMemprofChunkByAddr((uptr)p, user_requested_size); + if (!m) + return nullptr; + if (user_requested_size == 0) + return nullptr; + + return (const void *)m->Beg(); +} + +uptr memprof_malloc_usable_size(const void *ptr) { + if (!ptr) + return 0; + uptr usable_size = instance.AllocationSize(reinterpret_cast<uptr>(ptr)); + return usable_size; +} + +} // namespace __memprof + +// ---------------------- Interface ---------------- {{{1 +using namespace __memprof; + +uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } + +int __sanitizer_get_ownership(const void *p) { + return memprof_malloc_usable_size(p) != 0; +} + +const void *__sanitizer_get_allocated_begin(const void *p) { + return memprof_malloc_begin(p); +} + +uptr __sanitizer_get_allocated_size(const void *p) { + return memprof_malloc_usable_size(p); +} + +uptr __sanitizer_get_allocated_size_fast(const void *p) { + DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); + uptr ret = instance.AllocationSizeFast(reinterpret_cast<uptr>(p)); + DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); + return ret; +} + +void __sanitizer_purge_allocator() { instance.Purge(); } + +int __memprof_profile_dump() { + instance.FinishAndWrite(); + // In the future we may want to return non-zero if there are any errors + // detected during the dumping process. + return 0; +} + +void __memprof_profile_reset() { + if (report_file.fd != kInvalidFd && report_file.fd != kStdoutFd && + report_file.fd != kStderrFd) { + CloseFile(report_file.fd); + // Setting the file descriptor to kInvalidFd ensures that we will reopen the + // file when invoking Write again. + report_file.fd = kInvalidFd; + } +} diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_allocator.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_allocator.h new file mode 100644 index 000000000000..6d898f06f7e4 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_allocator.h @@ -0,0 +1,107 @@ +//===-- memprof_allocator.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header for memprof_allocator.cpp. +//===----------------------------------------------------------------------===// + +#ifndef MEMPROF_ALLOCATOR_H +#define MEMPROF_ALLOCATOR_H + +#include "memprof_flags.h" +#include "memprof_interceptors.h" +#include "memprof_internal.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_list.h" + +#if !defined(__x86_64__) +#error Unsupported platform +#endif +#if !SANITIZER_CAN_USE_ALLOCATOR64 +#error Only 64-bit allocator supported +#endif + +namespace __memprof { + +enum AllocType { + FROM_MALLOC = 1, // Memory block came from malloc, calloc, realloc, etc. + FROM_NEW = 2, // Memory block came from operator new. + FROM_NEW_BR = 3 // Memory block came from operator new [ ] +}; + +void InitializeAllocator(); + +struct MemprofMapUnmapCallback { + void OnMap(uptr p, uptr size) const; + void OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const { + OnMap(p, size); + } + void OnUnmap(uptr p, uptr size) const; +}; + +constexpr uptr kAllocatorSpace = ~(uptr)0; +constexpr uptr kAllocatorSize = 0x40000000000ULL; // 4T. +typedef DefaultSizeClassMap SizeClassMap; +template <typename AddressSpaceViewTy> +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = kAllocatorSize; + static const uptr kMetadataSize = 0; + typedef __memprof::SizeClassMap SizeClassMap; + typedef MemprofMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; +}; + +template <typename AddressSpaceView> +using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>; +using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; + +static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; + +template <typename AddressSpaceView> +using MemprofAllocatorASVT = + CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>; +using MemprofAllocator = MemprofAllocatorASVT<LocalAddressSpaceView>; +using AllocatorCache = MemprofAllocator::AllocatorCache; + +struct MemprofThreadLocalMallocStorage { + AllocatorCache allocator_cache; + void CommitBack(); + +private: + // These objects are allocated via mmap() and are zero-initialized. + MemprofThreadLocalMallocStorage() {} +}; + +void *memprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, + AllocType alloc_type); +void memprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type); +void memprof_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type); + +void *memprof_malloc(uptr size, BufferedStackTrace *stack); +void *memprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack); +void *memprof_realloc(void *p, uptr size, BufferedStackTrace *stack); +void *memprof_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack); +void *memprof_valloc(uptr size, BufferedStackTrace *stack); +void *memprof_pvalloc(uptr size, BufferedStackTrace *stack); + +void *memprof_aligned_alloc(uptr alignment, uptr size, + BufferedStackTrace *stack); +int memprof_posix_memalign(void **memptr, uptr alignment, uptr size, + BufferedStackTrace *stack); +uptr memprof_malloc_usable_size(const void *ptr); + +void PrintInternalAllocatorStats(); + +} // namespace __memprof +#endif // MEMPROF_ALLOCATOR_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_descriptions.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_descriptions.cpp new file mode 100644 index 000000000000..4fbe2943e653 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_descriptions.cpp @@ -0,0 +1,70 @@ +//===-- memprof_descriptions.cpp -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf functions for getting information about an address and/or printing +// it. +//===----------------------------------------------------------------------===// + +#include "memprof_descriptions.h" +#include "memprof_mapping.h" +#include "memprof_stack.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + +namespace __memprof { + +MemprofThreadIdAndName::MemprofThreadIdAndName(MemprofThreadContext *t) { + Init(t->tid, t->name); +} + +MemprofThreadIdAndName::MemprofThreadIdAndName(u32 tid) { + if (tid == kInvalidTid) { + Init(tid, ""); + } else { + memprofThreadRegistry().CheckLocked(); + MemprofThreadContext *t = GetThreadContextByTidLocked(tid); + Init(tid, t->name); + } +} + +void MemprofThreadIdAndName::Init(u32 tid, const char *tname) { + int len = internal_snprintf(name, sizeof(name), "T%d", tid); + CHECK(((unsigned int)len) < sizeof(name)); + if (tname[0] != '\0') + internal_snprintf(&name[len], sizeof(name) - len, " (%s)", tname); +} + +void DescribeThread(MemprofThreadContext *context) { + CHECK(context); + memprofThreadRegistry().CheckLocked(); + // No need to announce the main thread. + if (context->tid == kMainTid || context->announced) { + return; + } + context->announced = true; + InternalScopedString str; + str.AppendF("Thread %s", MemprofThreadIdAndName(context).c_str()); + if (context->parent_tid == kInvalidTid) { + str.Append(" created by unknown thread\n"); + Printf("%s", str.data()); + return; + } + str.AppendF(" created by %s here:\n", + MemprofThreadIdAndName(context->parent_tid).c_str()); + Printf("%s", str.data()); + StackDepotGet(context->stack_id).Print(); + // Recursively described parent thread if needed. + if (flags()->print_full_thread_history) { + MemprofThreadContext *parent_context = + GetThreadContextByTidLocked(context->parent_tid); + DescribeThread(parent_context); + } +} + +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_descriptions.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_descriptions.h new file mode 100644 index 000000000000..e88ea441bf9e --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_descriptions.h @@ -0,0 +1,45 @@ +//===-- memprof_descriptions.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header for memprof_descriptions.cpp. +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_DESCRIPTIONS_H +#define MEMPROF_DESCRIPTIONS_H + +#include "memprof_allocator.h" +#include "memprof_thread.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_report_decorator.h" + +namespace __memprof { + +void DescribeThread(MemprofThreadContext *context); +inline void DescribeThread(MemprofThread *t) { + if (t) + DescribeThread(t->context()); +} + +class MemprofThreadIdAndName { +public: + explicit MemprofThreadIdAndName(MemprofThreadContext *t); + explicit MemprofThreadIdAndName(u32 tid); + + // Contains "T%tid (%name)" or "T%tid" if the name is empty. + const char *c_str() const { return &name[0]; } + +private: + void Init(u32 tid, const char *tname); + + char name[128]; +}; + +} // namespace __memprof + +#endif // MEMPROF_DESCRIPTIONS_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.cpp new file mode 100644 index 000000000000..b107ff8fa0a7 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.cpp @@ -0,0 +1,93 @@ +//===-- memprof_flags.cpp --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf flag parsing logic. +//===----------------------------------------------------------------------===// + +#include "memprof_flags.h" +#include "memprof_interface_internal.h" +#include "memprof_stack.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_flags.h" + +namespace __memprof { + +Flags memprof_flags_dont_use_directly; // use via flags(). + +static const char *MaybeUseMemprofDefaultOptionsCompileDefinition() { +#ifdef MEMPROF_DEFAULT_OPTIONS + return SANITIZER_STRINGIFY(MEMPROF_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +void Flags::SetDefaults() { +#define MEMPROF_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "memprof_flags.inc" +#undef MEMPROF_FLAG +} + +static void RegisterMemprofFlags(FlagParser *parser, Flags *f) { +#define MEMPROF_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "memprof_flags.inc" +#undef MEMPROF_FLAG +} + +void InitializeFlags() { + // Set the default values and prepare for parsing MemProf and common flags. + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.external_symbolizer_path = GetEnv("MEMPROF_SYMBOLIZER_PATH"); + cf.malloc_context_size = kDefaultMallocContextSize; + cf.intercept_tls_get_addr = true; + cf.exitcode = 1; + OverrideCommonFlags(cf); + } + Flags *f = flags(); + f->SetDefaults(); + + FlagParser memprof_parser; + RegisterMemprofFlags(&memprof_parser, f); + RegisterCommonFlags(&memprof_parser); + + // Override from MemProf compile definition. + const char *memprof_compile_def = + MaybeUseMemprofDefaultOptionsCompileDefinition(); + memprof_parser.ParseString(memprof_compile_def); + + // Override from user-specified string. + const char *memprof_default_options = __memprof_default_options(); + memprof_parser.ParseString(memprof_default_options); + + // Override from command line. + memprof_parser.ParseStringFromEnv("MEMPROF_OPTIONS"); + + InitializeCommonFlags(); + + if (Verbosity()) + ReportUnrecognizedFlags(); + + if (common_flags()->help) { + memprof_parser.PrintFlagDescriptions(); + } + + CHECK_LE((uptr)common_flags()->malloc_context_size, kStackTraceMax); +} + +} // namespace __memprof + +SANITIZER_INTERFACE_WEAK_DEF(const char *, __memprof_default_options, void) { + return ""; +} diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.h new file mode 100644 index 000000000000..2f2b628653dc --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.h @@ -0,0 +1,45 @@ +//===-- memprof_flags.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf runtime flags. +//===----------------------------------------------------------------------===// + +#ifndef MEMPROF_FLAGS_H +#define MEMPROF_FLAGS_H + +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_internal_defs.h" + +// MemProf flag values can be defined in four ways: +// 1) initialized with default values at startup. +// 2) overriden during compilation of MemProf runtime by providing +// compile definition MEMPROF_DEFAULT_OPTIONS. +// 3) overriden from string returned by user-specified function +// __memprof_default_options(). +// 4) overriden from env variable MEMPROF_OPTIONS. + +namespace __memprof { + +struct Flags { +#define MEMPROF_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "memprof_flags.inc" +#undef MEMPROF_FLAG + + void SetDefaults(); +}; + +extern Flags memprof_flags_dont_use_directly; +inline Flags *flags() { return &memprof_flags_dont_use_directly; } + +void InitializeFlags(); + +} // namespace __memprof + +#endif // MEMPROF_FLAGS_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.inc b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.inc new file mode 100644 index 000000000000..7c5dc091f793 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_flags.inc @@ -0,0 +1,41 @@ +//===-- memprof_flags.inc --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MemProf runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_FLAG +#error "Define MEMPROF_FLAG prior to including this file!" +#endif + +// MEMPROF_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +MEMPROF_FLAG(bool, unmap_shadow_on_exit, false, + "If set, explicitly unmaps the (huge) shadow at exit.") +MEMPROF_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap") +MEMPROF_FLAG(bool, print_legend, true, "Print the legend for the shadow bytes.") +MEMPROF_FLAG(bool, atexit, false, + "If set, prints MemProf exit stats even after program terminates " + "successfully.") +MEMPROF_FLAG( + bool, print_full_thread_history, true, + "If set, prints thread creation stacks for the threads involved in the " + "report and their ancestors up to the main thread.") + +MEMPROF_FLAG(bool, halt_on_error, true, + "Crash the program after printing the first error report " + "(WARNING: USE AT YOUR OWN RISK!)") +MEMPROF_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true, + "realloc(p, 0) is equivalent to free(p) by default (Same as the " + "POSIX standard). If set to false, realloc(p, 0) will return a " + "pointer to an allocated space which can not be used.") +MEMPROF_FLAG(bool, print_text, false, + "If set, prints the heap profile in text format. Else use the raw binary serialization format.") +MEMPROF_FLAG(bool, print_terse, false, + "If set, prints memory profile in a terse format. Only applicable if print_text = true.")
\ No newline at end of file diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_init_version.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_init_version.h new file mode 100644 index 000000000000..26c68f78677a --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_init_version.h @@ -0,0 +1,26 @@ +//===-- memprof_init_version.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// This header defines a versioned __memprof_init function to be called at the +// startup of the instrumented program. +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_INIT_VERSION_H +#define MEMPROF_INIT_VERSION_H + +#include "sanitizer_common/sanitizer_platform.h" + +extern "C" { +// Every time the Memprof ABI changes we also change the version number in the +// __memprof_init function name. Objects built with incompatible Memprof ABI +// versions will not link with run-time. +#define __memprof_version_mismatch_check __memprof_version_mismatch_check_v1 +} + +#endif // MEMPROF_INIT_VERSION_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors.cpp new file mode 100644 index 000000000000..53ee4e953419 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors.cpp @@ -0,0 +1,341 @@ +//===-- memprof_interceptors.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Intercept various libc functions. +//===----------------------------------------------------------------------===// + +#include "memprof_interceptors.h" +#include "memprof_allocator.h" +#include "memprof_internal.h" +#include "memprof_mapping.h" +#include "memprof_stack.h" +#include "memprof_stats.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_posix.h" + +namespace __memprof { + +#define MEMPROF_READ_STRING(s, n) MEMPROF_READ_RANGE((s), (n)) + +static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) { +#if SANITIZER_INTERCEPT_STRNLEN + if (REAL(strnlen)) { + return REAL(strnlen)(s, maxlen); + } +#endif + return internal_strnlen(s, maxlen); +} + +void SetThreadName(const char *name) { + MemprofThread *t = GetCurrentThread(); + if (t) + memprofThreadRegistry().SetThreadName(t->tid(), name); +} + +int OnExit() { + // FIXME: ask frontend whether we need to return failure. + return 0; +} + +} // namespace __memprof + +// ---------------------- Wrappers ---------------- {{{1 +using namespace __memprof; + +DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr) +DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) + +#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ + MEMPROF_INTERCEPT_FUNC_VER(name, ver) +#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ + MEMPROF_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + MEMPROF_WRITE_RANGE(ptr, size) +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + MEMPROF_READ_RANGE(ptr, size) +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + MEMPROF_INTERCEPTOR_ENTER(ctx, func); \ + do { \ + if (memprof_init_is_running) \ + return REAL(func)(__VA_ARGS__); \ + ENSURE_MEMPROF_INITED(); \ + } while (false) +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) SetThreadName(name) +// Should be memprofThreadRegistry().SetThreadNameByUserId(thread, name) +// But memprof does not remember UserId's for threads (pthread_t); +// and remembers all ever existed threads, so the linear search by UserId +// can be slow. +#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) +#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) +#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() +#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!memprof_inited) +#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ + if (MemprofThread *t = GetCurrentThread()) { \ + *begin = t->tls_begin(); \ + *end = t->tls_end(); \ + } else { \ + *begin = *end = 0; \ + } + +#include "sanitizer_common/sanitizer_common_interceptors.inc" + +#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) MEMPROF_READ_RANGE(p, s) +#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) MEMPROF_WRITE_RANGE(p, s) +#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#include "sanitizer_common/sanitizer_common_syscalls.inc" + +struct ThreadStartParam { + atomic_uintptr_t t; + atomic_uintptr_t is_registered; +}; + +static thread_return_t THREAD_CALLING_CONV memprof_thread_start(void *arg) { + ThreadStartParam *param = reinterpret_cast<ThreadStartParam *>(arg); + MemprofThread *t = nullptr; + while ((t = reinterpret_cast<MemprofThread *>( + atomic_load(¶m->t, memory_order_acquire))) == nullptr) + internal_sched_yield(); + SetCurrentThread(t); + return t->ThreadStart(GetTid(), ¶m->is_registered); +} + +INTERCEPTOR(int, pthread_create, void *thread, void *attr, + void *(*start_routine)(void *), void *arg) { + EnsureMainThreadIDIsCorrect(); + GET_STACK_TRACE_THREAD; + int detached = 0; + if (attr) + REAL(pthread_attr_getdetachstate)(attr, &detached); + ThreadStartParam param; + atomic_store(¶m.t, 0, memory_order_relaxed); + atomic_store(¶m.is_registered, 0, memory_order_relaxed); + int result; + { + // Ignore all allocations made by pthread_create: thread stack/TLS may be + // stored by pthread for future reuse even after thread destruction, and + // the linked list it's stored in doesn't even hold valid pointers to the + // objects, the latter are calculated by obscure pointer arithmetic. + result = REAL(pthread_create)(thread, attr, memprof_thread_start, ¶m); + } + if (result == 0) { + u32 current_tid = GetCurrentTidOrInvalid(); + MemprofThread *t = MemprofThread::Create(start_routine, arg, current_tid, + &stack, detached); + atomic_store(¶m.t, reinterpret_cast<uptr>(t), memory_order_release); + // Wait until the MemprofThread object is initialized and the + // ThreadRegistry entry is in "started" state. + while (atomic_load(¶m.is_registered, memory_order_acquire) == 0) + internal_sched_yield(); + } + return result; +} + +INTERCEPTOR(int, pthread_join, void *t, void **arg) { + return REAL(pthread_join)(t, arg); +} + +DEFINE_INTERNAL_PTHREAD_FUNCTIONS + +INTERCEPTOR(char *, index, const char *string, int c) +ALIAS(WRAP(strchr)); + +// For both strcat() and strncat() we need to check the validity of |to| +// argument irrespective of the |from| length. +INTERCEPTOR(char *, strcat, char *to, const char *from) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strcat); + ENSURE_MEMPROF_INITED(); + uptr from_length = internal_strlen(from); + MEMPROF_READ_RANGE(from, from_length + 1); + uptr to_length = internal_strlen(to); + MEMPROF_READ_STRING(to, to_length); + MEMPROF_WRITE_RANGE(to + to_length, from_length + 1); + return REAL(strcat)(to, from); +} + +INTERCEPTOR(char *, strncat, char *to, const char *from, uptr size) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strncat); + ENSURE_MEMPROF_INITED(); + uptr from_length = MaybeRealStrnlen(from, size); + uptr copy_length = Min(size, from_length + 1); + MEMPROF_READ_RANGE(from, copy_length); + uptr to_length = internal_strlen(to); + MEMPROF_READ_STRING(to, to_length); + MEMPROF_WRITE_RANGE(to + to_length, from_length + 1); + return REAL(strncat)(to, from, size); +} + +INTERCEPTOR(char *, strcpy, char *to, const char *from) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strcpy); + if (memprof_init_is_running) { + return REAL(strcpy)(to, from); + } + ENSURE_MEMPROF_INITED(); + uptr from_size = internal_strlen(from) + 1; + MEMPROF_READ_RANGE(from, from_size); + MEMPROF_WRITE_RANGE(to, from_size); + return REAL(strcpy)(to, from); +} + +INTERCEPTOR(char *, strdup, const char *s) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strdup); + if (UNLIKELY(!memprof_inited)) + return internal_strdup(s); + ENSURE_MEMPROF_INITED(); + uptr length = internal_strlen(s); + MEMPROF_READ_RANGE(s, length + 1); + GET_STACK_TRACE_MALLOC; + void *new_mem = memprof_malloc(length + 1, &stack); + REAL(memcpy)(new_mem, s, length + 1); + return reinterpret_cast<char *>(new_mem); +} + +INTERCEPTOR(char *, __strdup, const char *s) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strdup); + if (UNLIKELY(!memprof_inited)) + return internal_strdup(s); + ENSURE_MEMPROF_INITED(); + uptr length = internal_strlen(s); + MEMPROF_READ_RANGE(s, length + 1); + GET_STACK_TRACE_MALLOC; + void *new_mem = memprof_malloc(length + 1, &stack); + REAL(memcpy)(new_mem, s, length + 1); + return reinterpret_cast<char *>(new_mem); +} + +INTERCEPTOR(char *, strncpy, char *to, const char *from, uptr size) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strncpy); + ENSURE_MEMPROF_INITED(); + uptr from_size = Min(size, MaybeRealStrnlen(from, size) + 1); + MEMPROF_READ_RANGE(from, from_size); + MEMPROF_WRITE_RANGE(to, size); + return REAL(strncpy)(to, from, size); +} + +INTERCEPTOR(long, strtol, const char *nptr, char **endptr, int base) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strtol); + ENSURE_MEMPROF_INITED(); + char *real_endptr; + long result = REAL(strtol)(nptr, &real_endptr, base); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + return result; +} + +INTERCEPTOR(int, atoi, const char *nptr) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, atoi); + ENSURE_MEMPROF_INITED(); + char *real_endptr; + // "man atoi" tells that behavior of atoi(nptr) is the same as + // strtol(nptr, 0, 10), i.e. it sets errno to ERANGE if the + // parsed integer can't be stored in *long* type (even if it's + // different from int). So, we just imitate this behavior. + int result = REAL(strtol)(nptr, &real_endptr, 10); + FixRealStrtolEndptr(nptr, &real_endptr); + MEMPROF_READ_STRING(nptr, (real_endptr - nptr) + 1); + return result; +} + +INTERCEPTOR(long, atol, const char *nptr) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, atol); + ENSURE_MEMPROF_INITED(); + char *real_endptr; + long result = REAL(strtol)(nptr, &real_endptr, 10); + FixRealStrtolEndptr(nptr, &real_endptr); + MEMPROF_READ_STRING(nptr, (real_endptr - nptr) + 1); + return result; +} + +INTERCEPTOR(long long, strtoll, const char *nptr, char **endptr, int base) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, strtoll); + ENSURE_MEMPROF_INITED(); + char *real_endptr; + long long result = REAL(strtoll)(nptr, &real_endptr, base); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + return result; +} + +INTERCEPTOR(long long, atoll, const char *nptr) { + void *ctx; + MEMPROF_INTERCEPTOR_ENTER(ctx, atoll); + ENSURE_MEMPROF_INITED(); + char *real_endptr; + long long result = REAL(strtoll)(nptr, &real_endptr, 10); + FixRealStrtolEndptr(nptr, &real_endptr); + MEMPROF_READ_STRING(nptr, (real_endptr - nptr) + 1); + return result; +} + +// ---------------------- InitializeMemprofInterceptors ---------------- {{{1 +namespace __memprof { +void InitializeMemprofInterceptors() { + static bool was_called_once; + CHECK(!was_called_once); + was_called_once = true; + InitializeCommonInterceptors(); + + // Intercept str* functions. + MEMPROF_INTERCEPT_FUNC(strcat); + MEMPROF_INTERCEPT_FUNC(strcpy); + MEMPROF_INTERCEPT_FUNC(strncat); + MEMPROF_INTERCEPT_FUNC(strncpy); + MEMPROF_INTERCEPT_FUNC(strdup); + MEMPROF_INTERCEPT_FUNC(__strdup); + MEMPROF_INTERCEPT_FUNC(index); + + MEMPROF_INTERCEPT_FUNC(atoi); + MEMPROF_INTERCEPT_FUNC(atol); + MEMPROF_INTERCEPT_FUNC(strtol); + MEMPROF_INTERCEPT_FUNC(atoll); + MEMPROF_INTERCEPT_FUNC(strtoll); + + // Intercept threading-related functions + MEMPROF_INTERCEPT_FUNC(pthread_create); + MEMPROF_INTERCEPT_FUNC(pthread_join); + + InitializePlatformInterceptors(); + + VReport(1, "MemProfiler: libc interceptors initialized\n"); +} + +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors.h new file mode 100644 index 000000000000..20edef42a515 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors.h @@ -0,0 +1,66 @@ +//===-- memprof_interceptors.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header for memprof_interceptors.cpp +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_INTERCEPTORS_H +#define MEMPROF_INTERCEPTORS_H + +#include "interception/interception.h" +#include "memprof_interceptors_memintrinsics.h" +#include "memprof_internal.h" +#include "sanitizer_common/sanitizer_platform_interceptors.h" + +namespace __memprof { + +void InitializeMemprofInterceptors(); +void InitializePlatformInterceptors(); + +#define ENSURE_MEMPROF_INITED() \ + do { \ + CHECK(!memprof_init_is_running); \ + if (UNLIKELY(!memprof_inited)) { \ + MemprofInitFromRtl(); \ + } \ + } while (0) + +} // namespace __memprof + +DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size) +DECLARE_REAL(char *, strchr, const char *str, int c) +DECLARE_REAL(SIZE_T, strlen, const char *s) +DECLARE_REAL(char *, strncpy, char *to, const char *from, uptr size) +DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen) +DECLARE_REAL(char *, strstr, const char *s1, const char *s2) + +#define MEMPROF_INTERCEPT_FUNC(name) \ + do { \ + if (!INTERCEPT_FUNCTION(name)) \ + VReport(1, "MemProfiler: failed to intercept '%s'\n'", #name); \ + } while (0) +#define MEMPROF_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if (!INTERCEPT_FUNCTION_VER(name, ver)) \ + VReport(1, "MemProfiler: failed to intercept '%s@@%s'\n", #name, ver); \ + } while (0) +#define MEMPROF_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \ + do { \ + if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \ + VReport(1, "MemProfiler: failed to intercept '%s@@%s' or '%s'\n", #name, \ + ver, #name); \ + } while (0) + +#define MEMPROF_INTERCEPTOR_ENTER(ctx, func) \ + ctx = 0; \ + (void)ctx; + +#define COMMON_INTERCEPT_FUNCTION(name) MEMPROF_INTERCEPT_FUNC(name) + +#endif // MEMPROF_INTERCEPTORS_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.cpp new file mode 100644 index 000000000000..56bd11614d6a --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.cpp @@ -0,0 +1,92 @@ +//===-- memprof_interceptors_memintrinsics.cpp ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf versions of memcpy, memmove, and memset. +//===---------------------------------------------------------------------===// + +#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS + +#include "memprof_interceptors_memintrinsics.h" + +#include "memprof_interceptors.h" +#include "memprof_stack.h" + +using namespace __memprof; + +// memcpy is called during __memprof_init() from the internals of printf(...). +// We do not treat memcpy with to==from as a bug. +// See http://llvm.org/bugs/show_bug.cgi?id=11763. +#define MEMPROF_MEMCPY_IMPL(to, from, size) \ + do { \ + if (UNLIKELY(!memprof_inited)) \ + return internal_memcpy(to, from, size); \ + if (memprof_init_is_running) { \ + return REAL(memcpy)(to, from, size); \ + } \ + ENSURE_MEMPROF_INITED(); \ + MEMPROF_READ_RANGE(from, size); \ + MEMPROF_WRITE_RANGE(to, size); \ + return REAL(memcpy)(to, from, size); \ + } while (0) + +// memset is called inside Printf. +#define MEMPROF_MEMSET_IMPL(block, c, size) \ + do { \ + if (UNLIKELY(!memprof_inited)) \ + return internal_memset(block, c, size); \ + if (memprof_init_is_running) { \ + return REAL(memset)(block, c, size); \ + } \ + ENSURE_MEMPROF_INITED(); \ + MEMPROF_WRITE_RANGE(block, size); \ + return REAL(memset)(block, c, size); \ + } while (0) + +#define MEMPROF_MEMMOVE_IMPL(to, from, size) \ + do { \ + if (UNLIKELY(!memprof_inited)) \ + return internal_memmove(to, from, size); \ + ENSURE_MEMPROF_INITED(); \ + MEMPROF_READ_RANGE(from, size); \ + MEMPROF_WRITE_RANGE(to, size); \ + return internal_memmove(to, from, size); \ + } while (0) + +#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + MEMPROF_INTERCEPTOR_ENTER(ctx, memmove); \ + MEMPROF_MEMMOVE_IMPL(to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + MEMPROF_INTERCEPTOR_ENTER(ctx, memcpy); \ + MEMPROF_MEMCPY_IMPL(to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + MEMPROF_INTERCEPTOR_ENTER(ctx, memset); \ + MEMPROF_MEMSET_IMPL(block, c, size); \ + } while (false) + +#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc" + +void *__memprof_memcpy(void *to, const void *from, uptr size) { + MEMPROF_MEMCPY_IMPL(to, from, size); +} + +void *__memprof_memset(void *block, int c, uptr size) { + MEMPROF_MEMSET_IMPL(block, c, size); +} + +void *__memprof_memmove(void *to, const void *from, uptr size) { + MEMPROF_MEMMOVE_IMPL(to, from, size); +} diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h new file mode 100644 index 000000000000..0b87a6f3522a --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h @@ -0,0 +1,40 @@ +//===-- memprof_interceptors_memintrinsics.h -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header for memprof_interceptors_memintrinsics.cpp +//===---------------------------------------------------------------------===// +#ifndef MEMPROF_MEMINTRIN_H +#define MEMPROF_MEMINTRIN_H + +#include "interception/interception.h" +#include "memprof_interface_internal.h" +#include "memprof_internal.h" +#include "memprof_mapping.h" + +DECLARE_REAL(void *, memcpy, void *to, const void *from, uptr size) +DECLARE_REAL(void *, memset, void *block, int c, uptr size) + +namespace __memprof { + +// We implement ACCESS_MEMORY_RANGE, MEMPROF_READ_RANGE, +// and MEMPROF_WRITE_RANGE as macro instead of function so +// that no extra frames are created, and stack trace contains +// relevant information only. +#define ACCESS_MEMORY_RANGE(offset, size) \ + do { \ + __memprof_record_access_range(offset, size); \ + } while (0) + +#define MEMPROF_READ_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size) +#define MEMPROF_WRITE_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size) + +} // namespace __memprof + +#endif // MEMPROF_MEMINTRIN_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interface_internal.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interface_internal.h new file mode 100644 index 000000000000..318bc4104405 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_interface_internal.h @@ -0,0 +1,65 @@ +//===-- memprof_interface_internal.h ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// This header declares the MemProfiler runtime interface functions. +// The runtime library has to define these functions so the instrumented program +// could call them. +// +// See also include/sanitizer/memprof_interface.h +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_INTERFACE_INTERNAL_H +#define MEMPROF_INTERFACE_INTERNAL_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +#include "memprof_init_version.h" + +using __sanitizer::u32; +using __sanitizer::u64; +using __sanitizer::uptr; + +extern "C" { +// This function should be called at the very beginning of the process, +// before any instrumented code is executed and before any call to malloc. +SANITIZER_INTERFACE_ATTRIBUTE void __memprof_init(); +SANITIZER_INTERFACE_ATTRIBUTE void __memprof_preinit(); +SANITIZER_INTERFACE_ATTRIBUTE void __memprof_version_mismatch_check_v1(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __memprof_record_access(void const volatile *addr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __memprof_record_access_range(void const volatile *addr, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE void __memprof_print_accumulated_stats(); + +SANITIZER_INTERFACE_ATTRIBUTE +const char *__memprof_default_options(); + +SANITIZER_INTERFACE_ATTRIBUTE +extern uptr __memprof_shadow_memory_dynamic_address; + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE extern char + __memprof_profile_filename[1]; +SANITIZER_INTERFACE_ATTRIBUTE int __memprof_profile_dump(); +SANITIZER_INTERFACE_ATTRIBUTE void __memprof_profile_reset(); + +SANITIZER_INTERFACE_ATTRIBUTE void __memprof_load(uptr p); +SANITIZER_INTERFACE_ATTRIBUTE void __memprof_store(uptr p); + +SANITIZER_INTERFACE_ATTRIBUTE +void *__memprof_memcpy(void *dst, const void *src, uptr size); +SANITIZER_INTERFACE_ATTRIBUTE +void *__memprof_memset(void *s, int c, uptr n); +SANITIZER_INTERFACE_ATTRIBUTE +void *__memprof_memmove(void *dest, const void *src, uptr n); +} // extern "C" + +#endif // MEMPROF_INTERFACE_INTERNAL_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_internal.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_internal.h new file mode 100644 index 000000000000..ec9fa10badec --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_internal.h @@ -0,0 +1,81 @@ +//===-- memprof_internal.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header which defines various general utilities. +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_INTERNAL_H +#define MEMPROF_INTERNAL_H + +#include "memprof_flags.h" +#include "memprof_interface_internal.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_stacktrace.h" + +// Build-time configuration options. + +// If set, memprof will intercept C++ exception api call(s). +#ifndef MEMPROF_HAS_EXCEPTIONS +#define MEMPROF_HAS_EXCEPTIONS 1 +#endif + +#ifndef MEMPROF_DYNAMIC +#ifdef PIC +#define MEMPROF_DYNAMIC 1 +#else +#define MEMPROF_DYNAMIC 0 +#endif +#endif + +// All internal functions in memprof reside inside the __memprof namespace +// to avoid namespace collisions with the user programs. +// Separate namespace also makes it simpler to distinguish the memprof +// run-time functions from the instrumented user code in a profile. +namespace __memprof { + +class MemprofThread; +using __sanitizer::StackTrace; + +void MemprofInitFromRtl(); + +// memprof_rtl.cpp +void PrintAddressSpaceLayout(); + +// memprof_shadow_setup.cpp +void InitializeShadowMemory(); + +// memprof_malloc_linux.cpp +void ReplaceSystemMalloc(); + +// memprof_linux.cpp +uptr FindDynamicShadowStart(); + +// memprof_thread.cpp +MemprofThread *CreateMainThread(); + +// Wrapper for TLS/TSD. +void TSDInit(void (*destructor)(void *tsd)); +void *TSDGet(); +void TSDSet(void *tsd); +void PlatformTSDDtor(void *tsd); + +void *MemprofDlSymNext(const char *sym); + +extern int memprof_inited; +extern int memprof_timestamp_inited; +// Used to avoid infinite recursion in __memprof_init(). +extern bool memprof_init_is_running; +extern void (*death_callback)(void); +extern long memprof_init_timestamp_s; + +} // namespace __memprof + +#endif // MEMPROF_INTERNAL_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_linux.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_linux.cpp new file mode 100644 index 000000000000..fbe5d250f840 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_linux.cpp @@ -0,0 +1,67 @@ +//===-- memprof_linux.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Linux-specific details. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if !SANITIZER_LINUX +#error Unsupported OS +#endif + +#include "memprof_interceptors.h" +#include "memprof_internal.h" +#include "memprof_thread.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_procmaps.h" + +#include <dlfcn.h> +#include <fcntl.h> +#include <limits.h> +#include <link.h> +#include <pthread.h> +#include <stdio.h> +#include <sys/mman.h> +#include <sys/resource.h> +#include <sys/syscall.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/ucontext.h> +#include <unistd.h> +#include <unwind.h> + +typedef enum { + MEMPROF_RT_VERSION_UNDEFINED = 0, + MEMPROF_RT_VERSION_DYNAMIC, + MEMPROF_RT_VERSION_STATIC, +} memprof_rt_version_t; + +// FIXME: perhaps also store abi version here? +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +memprof_rt_version_t __memprof_rt_version; +} + +namespace __memprof { + +void InitializePlatformInterceptors() {} +void InitializePlatformExceptionHandlers() {} + +uptr FindDynamicShadowStart() { + uptr shadow_size_bytes = MemToShadowSize(kHighMemEnd); + return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE, + /*min_shadow_base_alignment*/ 0, kHighMemEnd, + GetMmapGranularity()); +} + +void *MemprofDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); } + +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_malloc_linux.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_malloc_linux.cpp new file mode 100644 index 000000000000..aba6295a4a04 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_malloc_linux.cpp @@ -0,0 +1,149 @@ +//===-- memprof_malloc_linux.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Linux-specific malloc interception. +// We simply define functions like malloc, free, realloc, etc. +// They will replace the corresponding libc functions automagically. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if !SANITIZER_LINUX +#error Unsupported OS +#endif + +#include "memprof_allocator.h" +#include "memprof_interceptors.h" +#include "memprof_internal.h" +#include "memprof_stack.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +// ---------------------- Replacement functions ---------------- {{{1 +using namespace __memprof; + +struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> { + static bool UseImpl() { return memprof_init_is_running; } +}; + +INTERCEPTOR(void, free, void *ptr) { + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); + GET_STACK_TRACE_FREE; + memprof_free(ptr, &stack, FROM_MALLOC); +} + +#if SANITIZER_INTERCEPT_CFREE +INTERCEPTOR(void, cfree, void *ptr) { + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); + GET_STACK_TRACE_FREE; + memprof_free(ptr, &stack, FROM_MALLOC); +} +#endif // SANITIZER_INTERCEPT_CFREE + +INTERCEPTOR(void *, malloc, uptr size) { + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); + ENSURE_MEMPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return memprof_malloc(size, &stack); +} + +INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) { + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); + ENSURE_MEMPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return memprof_calloc(nmemb, size, &stack); +} + +INTERCEPTOR(void *, realloc, void *ptr, uptr size) { + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); + ENSURE_MEMPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return memprof_realloc(ptr, size, &stack); +} + +#if SANITIZER_INTERCEPT_REALLOCARRAY +INTERCEPTOR(void *, reallocarray, void *ptr, uptr nmemb, uptr size) { + ENSURE_MEMPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return memprof_reallocarray(ptr, nmemb, size, &stack); +} +#endif // SANITIZER_INTERCEPT_REALLOCARRAY + +#if SANITIZER_INTERCEPT_MEMALIGN +INTERCEPTOR(void *, memalign, uptr boundary, uptr size) { + GET_STACK_TRACE_MALLOC; + return memprof_memalign(boundary, size, &stack, FROM_MALLOC); +} + +INTERCEPTOR(void *, __libc_memalign, uptr boundary, uptr size) { + GET_STACK_TRACE_MALLOC; + void *res = memprof_memalign(boundary, size, &stack, FROM_MALLOC); + DTLS_on_libc_memalign(res, size); + return res; +} +#endif // SANITIZER_INTERCEPT_MEMALIGN + +#if SANITIZER_INTERCEPT_ALIGNED_ALLOC +INTERCEPTOR(void *, aligned_alloc, uptr boundary, uptr size) { + GET_STACK_TRACE_MALLOC; + return memprof_aligned_alloc(boundary, size, &stack); +} +#endif // SANITIZER_INTERCEPT_ALIGNED_ALLOC + +INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { + return memprof_malloc_usable_size(ptr); +} + +#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO +// We avoid including malloc.h for portability reasons. +// man mallinfo says the fields are "long", but the implementation uses int. +// It doesn't matter much -- we just need to make sure that the libc's mallinfo +// is not called. +struct fake_mallinfo { + int x[10]; +}; + +INTERCEPTOR(struct fake_mallinfo, mallinfo, void) { + struct fake_mallinfo res; + REAL(memset)(&res, 0, sizeof(res)); + return res; +} + +INTERCEPTOR(int, mallopt, int cmd, int value) { return 0; } +#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO + +INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) { + GET_STACK_TRACE_MALLOC; + return memprof_posix_memalign(memptr, alignment, size, &stack); +} + +INTERCEPTOR(void *, valloc, uptr size) { + GET_STACK_TRACE_MALLOC; + return memprof_valloc(size, &stack); +} + +#if SANITIZER_INTERCEPT_PVALLOC +INTERCEPTOR(void *, pvalloc, uptr size) { + GET_STACK_TRACE_MALLOC; + return memprof_pvalloc(size, &stack); +} +#endif // SANITIZER_INTERCEPT_PVALLOC + +INTERCEPTOR(void, malloc_stats, void) { __memprof_print_accumulated_stats(); } + +namespace __memprof { +void ReplaceSystemMalloc() {} +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mapping.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mapping.h new file mode 100644 index 000000000000..fef8acfcfc92 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mapping.h @@ -0,0 +1,148 @@ +//===-- memprof_mapping.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Defines MemProf memory mapping. +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_MAPPING_H +#define MEMPROF_MAPPING_H + +#include "memprof_internal.h" + +static const u64 kDefaultShadowScale = 3; +#define SHADOW_SCALE kDefaultShadowScale + +#define SHADOW_OFFSET __memprof_shadow_memory_dynamic_address + +#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE) +#define MEMPROF_ALIGNMENT 32 +namespace __memprof { + +extern uptr kHighMemEnd; // Initialized in __memprof_init. + +} // namespace __memprof + +// Size of memory block mapped to a single shadow location +#define MEM_GRANULARITY 64ULL + +#define SHADOW_MASK ~(MEM_GRANULARITY - 1) + +#define MEM_TO_SHADOW(mem) \ + ((((mem) & SHADOW_MASK) >> SHADOW_SCALE) + (SHADOW_OFFSET)) + +// Histogram shadow memory is laid different to the standard configuration: + +// 8 bytes +// +---+---+---+ +---+---+---+ +---+---+---+ +// Memory | a | | b | | c | +// +---+---+---+ +---+---+---+ +---+---+---+ + +// +---+ +---+ +---+ +// Shadow | a | | b | | c | +// +---+ +---+ +---+ +// 1 byte +// +// Where we have a 1 byte counter for each 8 bytes. HISTOGRAM_MEM_TO_SHADOW +// translates a memory address to the address of its corresponding shadow +// counter memory address. The same data is still provided in MIB whether +// histograms are used or not. Total access counts per allocations are +// computed by summing up all individual 1 byte counters. This can incur an +// accuracy penalty. + +#define HISTOGRAM_GRANULARITY 8U + +#define HISTOGRAM_MAX_COUNTER 255U + +#define HISTOGRAM_SHADOW_MASK ~(HISTOGRAM_GRANULARITY - 1) + +#define HISTOGRAM_MEM_TO_SHADOW(mem) \ + ((((mem) & HISTOGRAM_SHADOW_MASK) >> SHADOW_SCALE) + (SHADOW_OFFSET)) + +#define SHADOW_ENTRY_SIZE (MEM_GRANULARITY >> SHADOW_SCALE) + +#define kLowMemBeg 0 +#define kLowMemEnd (SHADOW_OFFSET ? SHADOW_OFFSET - 1 : 0) + +#define kLowShadowBeg SHADOW_OFFSET +#define kLowShadowEnd (MEM_TO_SHADOW(kLowMemEnd) + SHADOW_ENTRY_SIZE - 1) + +#define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1 + SHADOW_ENTRY_SIZE - 1) + +#define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg) +#define kHighShadowEnd (MEM_TO_SHADOW(kHighMemEnd) + SHADOW_ENTRY_SIZE - 1) + +// With the zero shadow base we can not actually map pages starting from 0. +// This constant is somewhat arbitrary. +#define kZeroBaseShadowStart 0 +#define kZeroBaseMaxShadowStart (1 << 18) + +#define kShadowGapBeg (kLowShadowEnd ? kLowShadowEnd + 1 : kZeroBaseShadowStart) +#define kShadowGapEnd (kHighShadowBeg - 1) + +namespace __memprof { + +inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; } +inline bool AddrIsInLowMem(uptr a) { return a <= kLowMemEnd; } + +inline bool AddrIsInLowShadow(uptr a) { + return a >= kLowShadowBeg && a <= kLowShadowEnd; +} + +inline bool AddrIsInHighMem(uptr a) { + return kHighMemBeg && a >= kHighMemBeg && a <= kHighMemEnd; +} + +inline bool AddrIsInHighShadow(uptr a) { + return kHighMemBeg && a >= kHighShadowBeg && a <= kHighShadowEnd; +} + +inline bool AddrIsInShadowGap(uptr a) { + // In zero-based shadow mode we treat addresses near zero as addresses + // in shadow gap as well. + if (SHADOW_OFFSET == 0) + return a <= kShadowGapEnd; + return a >= kShadowGapBeg && a <= kShadowGapEnd; +} + +inline bool AddrIsInMem(uptr a) { + return AddrIsInLowMem(a) || AddrIsInHighMem(a) || + (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a)); +} + +inline uptr MemToShadow(uptr p) { + CHECK(AddrIsInMem(p)); + return MEM_TO_SHADOW(p); +} + +inline bool AddrIsInShadow(uptr a) { + return AddrIsInLowShadow(a) || AddrIsInHighShadow(a); +} + +inline bool AddrIsAlignedByGranularity(uptr a) { + return (a & (SHADOW_GRANULARITY - 1)) == 0; +} + +inline void RecordAccess(uptr a) { + // If we use a different shadow size then the type below needs adjustment. + CHECK_EQ(SHADOW_ENTRY_SIZE, 8); + u64 *shadow_address = (u64 *)MEM_TO_SHADOW(a); + (*shadow_address)++; +} + +inline void RecordAccessHistogram(uptr a) { + CHECK_EQ(SHADOW_ENTRY_SIZE, 8); + u8 *shadow_address = (u8 *)HISTOGRAM_MEM_TO_SHADOW(a); + if (*shadow_address < HISTOGRAM_MAX_COUNTER) { + (*shadow_address)++; + } +} + +} // namespace __memprof + +#endif // MEMPROF_MAPPING_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mibmap.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mibmap.cpp new file mode 100644 index 000000000000..a49ed8bf4fd6 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mibmap.cpp @@ -0,0 +1,48 @@ +//===-- memprof_mibmap.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +//===----------------------------------------------------------------------===// + +#include "memprof_mibmap.h" +#include "profile/MemProfData.inc" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_mutex.h" + +namespace __memprof { +using ::llvm::memprof::MemInfoBlock; + +void InsertOrMerge(const uptr Id, const MemInfoBlock &Block, MIBMapTy &Map) { + MIBMapTy::Handle h(&Map, static_cast<uptr>(Id), /*remove=*/false, + /*create=*/true); + if (h.created()) { + LockedMemInfoBlock *lmib = + (LockedMemInfoBlock *)InternalAlloc(sizeof(LockedMemInfoBlock)); + lmib->mutex.Init(); + lmib->mib = Block; + *h = lmib; + } else { + LockedMemInfoBlock *lmib = *h; + SpinMutexLock lock(&lmib->mutex); + uintptr_t ShorterHistogram; + if (Block.AccessHistogramSize > lmib->mib.AccessHistogramSize) + ShorterHistogram = lmib->mib.AccessHistogram; + else + ShorterHistogram = Block.AccessHistogram; + + lmib->mib.Merge(Block); + // The larger histogram is kept and the shorter histogram is discarded after + // adding the counters to the larger historam. Free only the shorter + // Histogram + if (Block.AccessHistogramSize > 0 || lmib->mib.AccessHistogramSize > 0) + InternalFree((void *)ShorterHistogram); + } +} + +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mibmap.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mibmap.h new file mode 100644 index 000000000000..a7cd420464e8 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_mibmap.h @@ -0,0 +1,27 @@ +#ifndef MEMPROF_MIBMAP_H_ +#define MEMPROF_MIBMAP_H_ + +#include <stdint.h> + +#include "profile/MemProfData.inc" +#include "sanitizer_common/sanitizer_addrhashmap.h" +#include "sanitizer_common/sanitizer_mutex.h" + +namespace __memprof { + +struct LockedMemInfoBlock { + __sanitizer::StaticSpinMutex mutex; + ::llvm::memprof::MemInfoBlock mib; +}; + +// The MIB map stores a mapping from stack ids to MemInfoBlocks. +typedef __sanitizer::AddrHashMap<LockedMemInfoBlock *, 200003> MIBMapTy; + +// Insert a new MemInfoBlock or merge with an existing block identified by the +// stack id. +void InsertOrMerge(const uptr Id, const ::llvm::memprof::MemInfoBlock &Block, + MIBMapTy &Map); + +} // namespace __memprof + +#endif // MEMPROF_MIBMAP_H_ diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_new_delete.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_new_delete.cpp new file mode 100644 index 000000000000..cae5de301367 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_new_delete.cpp @@ -0,0 +1,145 @@ +//===-- memprof_interceptors.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Interceptors for operators new and delete. +//===----------------------------------------------------------------------===// + +#include "memprof_allocator.h" +#include "memprof_internal.h" +#include "memprof_stack.h" +#include "sanitizer_common/sanitizer_allocator_report.h" + +#include "interception/interception.h" + +#include <stddef.h> + +#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE + +using namespace __memprof; + +// Fake std::nothrow_t and std::align_val_t to avoid including <new>. +namespace std { +struct nothrow_t {}; +enum class align_val_t : size_t {}; +} // namespace std + +#define OPERATOR_NEW_BODY(type, nothrow) \ + GET_STACK_TRACE_MALLOC; \ + void *res = memprof_memalign(0, size, &stack, type); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ + return res; +#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \ + GET_STACK_TRACE_MALLOC; \ + void *res = memprof_memalign((uptr)align, size, &stack, type); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ + return res; + +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size) { + OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size) { + OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::nothrow_t const &) { + OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::nothrow_t const &) { + OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW, true /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); +} + +#define OPERATOR_DELETE_BODY(type) \ + GET_STACK_TRACE_FREE; \ + memprof_delete(ptr, 0, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE(type) \ + GET_STACK_TRACE_FREE; \ + memprof_delete(ptr, size, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_ALIGN(type) \ + GET_STACK_TRACE_FREE; \ + memprof_delete(ptr, 0, static_cast<uptr>(align), &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \ + GET_STACK_TRACE_FREE; \ + memprof_delete(ptr, size, static_cast<uptr>(align), &stack, type); + +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr)NOEXCEPT { OPERATOR_DELETE_BODY(FROM_NEW); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr) NOEXCEPT { + OPERATOR_DELETE_BODY(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, size_t size)NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align)NOEXCEPT { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, size_t size, std::align_val_t align)NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size, + std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); +} diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_posix.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_posix.cpp new file mode 100644 index 000000000000..ee0821b85102 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_posix.cpp @@ -0,0 +1,55 @@ +//===-- memprof_posix.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Posix-specific details. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if !SANITIZER_POSIX +#error Only Posix supported +#endif + +#include "memprof_thread.h" +#include "sanitizer_common/sanitizer_internal_defs.h" + +#include <pthread.h> + +namespace __memprof { + +// ---------------------- TSD ---------------- {{{1 + +static pthread_key_t tsd_key; +static bool tsd_key_inited = false; +void TSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_key_inited); + tsd_key_inited = true; + CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); +} + +void *TSDGet() { + CHECK(tsd_key_inited); + return pthread_getspecific(tsd_key); +} + +void TSDSet(void *tsd) { + CHECK(tsd_key_inited); + pthread_setspecific(tsd_key, tsd); +} + +void PlatformTSDDtor(void *tsd) { + MemprofThreadContext *context = (MemprofThreadContext *)tsd; + if (context->destructor_iterations > 1) { + context->destructor_iterations--; + CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); + return; + } + MemprofThread::TSDDtor(tsd); +} +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_preinit.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_preinit.cpp new file mode 100644 index 000000000000..ad62a99c44c7 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_preinit.cpp @@ -0,0 +1,22 @@ +//===-- memprof_preinit.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Call __memprof_init at the very early stage of process startup. +//===----------------------------------------------------------------------===// +#include "memprof_internal.h" + +using namespace __memprof; + +#if SANITIZER_CAN_USE_PREINIT_ARRAY +// This section is linked into the main executable when -fmemory-profile is +// specified to perform initialization at a very early stage. +__attribute__((section(".preinit_array"), used)) static auto preinit = + __memprof_preinit; +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rawprofile.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rawprofile.cpp new file mode 100644 index 000000000000..a89764858482 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rawprofile.cpp @@ -0,0 +1,289 @@ +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "memprof_rawprofile.h" +#include "profile/MemProfData.inc" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_array_ref.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_stackdepotbase.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_vector.h" + +namespace __memprof { +using ::__sanitizer::Vector; +using ::llvm::memprof::MemInfoBlock; +using SegmentEntry = ::llvm::memprof::SegmentEntry; +using Header = ::llvm::memprof::Header; + +namespace { +template <class T> char *WriteBytes(const T &Pod, char *Buffer) { + *(T *)Buffer = Pod; + return Buffer + sizeof(T); +} + +void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, + void *Arg) { + // No need to touch the MIB value here since we are only recording the key. + auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg); + StackIds->PushBack(Key); +} +} // namespace + +u64 SegmentSizeBytes(ArrayRef<LoadedModule> Modules) { + u64 NumSegmentsToRecord = 0; + for (const auto &Module : Modules) { + for (const auto &Segment : Module.ranges()) { + if (Segment.executable) + NumSegmentsToRecord++; + } + } + + return sizeof(u64) // A header which stores the number of records. + + sizeof(SegmentEntry) * NumSegmentsToRecord; +} + +// The segment section uses the following format: +// ---------- Segment Info +// Num Entries +// ---------- Segment Entry +// Start +// End +// Offset +// UuidSize +// Uuid 32B +// ---------- +// ... +void SerializeSegmentsToBuffer(ArrayRef<LoadedModule> Modules, + const u64 ExpectedNumBytes, char *&Buffer) { + char *Ptr = Buffer; + // Reserve space for the final count. + Ptr += sizeof(u64); + + u64 NumSegmentsRecorded = 0; + + for (const auto &Module : Modules) { + for (const auto &Segment : Module.ranges()) { + if (Segment.executable) { + SegmentEntry Entry(Segment.beg, Segment.end, Module.base_address()); + CHECK(Module.uuid_size() <= MEMPROF_BUILDID_MAX_SIZE); + Entry.BuildIdSize = Module.uuid_size(); + memcpy(Entry.BuildId, Module.uuid(), Module.uuid_size()); + memcpy(Ptr, &Entry, sizeof(SegmentEntry)); + Ptr += sizeof(SegmentEntry); + NumSegmentsRecorded++; + } + } + } + // Store the number of segments we recorded in the space we reserved. + *((u64 *)Buffer) = NumSegmentsRecorded; + CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && + "Expected num bytes != actual bytes written"); +} + +u64 StackSizeBytes(const Vector<u64> &StackIds) { + u64 NumBytesToWrite = sizeof(u64); + + const u64 NumIds = StackIds.Size(); + for (unsigned k = 0; k < NumIds; ++k) { + const u64 Id = StackIds[k]; + // One entry for the id and then one more for the number of stack pcs. + NumBytesToWrite += 2 * sizeof(u64); + const StackTrace St = StackDepotGet(Id); + + CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace"); + for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) { + NumBytesToWrite += sizeof(u64); + } + } + return NumBytesToWrite; +} + +// The stack info section uses the following format: +// +// ---------- Stack Info +// Num Entries +// ---------- Stack Entry +// Num Stacks +// PC1 +// PC2 +// ... +// ---------- +void SerializeStackToBuffer(const Vector<u64> &StackIds, + const u64 ExpectedNumBytes, char *&Buffer) { + const u64 NumIds = StackIds.Size(); + char *Ptr = Buffer; + Ptr = WriteBytes(static_cast<u64>(NumIds), Ptr); + + for (unsigned k = 0; k < NumIds; ++k) { + const u64 Id = StackIds[k]; + Ptr = WriteBytes(Id, Ptr); + Ptr += sizeof(u64); // Bump it by u64, we will fill this in later. + u64 Count = 0; + const StackTrace St = StackDepotGet(Id); + for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) { + // PCs in stack traces are actually the return addresses, that is, + // addresses of the next instructions after the call. + uptr pc = StackTrace::GetPreviousInstructionPc(St.trace[i]); + Ptr = WriteBytes(static_cast<u64>(pc), Ptr); + ++Count; + } + // Store the count in the space we reserved earlier. + *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count; + } + + CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && + "Expected num bytes != actual bytes written"); +} + +// The MIB section has the following format: +// ---------- MIB Info +// Num Entries +// ---------- MIB Entry 0 +// Alloc Count +// ... +// ---- AccessHistogram Entry 0 +// ... +// ---- AccessHistogram Entry AccessHistogramSize - 1 +// ---------- MIB Entry 1 +// Alloc Count +// ... +// ---- AccessHistogram Entry 0 +// ... +// ---- AccessHistogram Entry AccessHistogramSize - 1 +// ---------- +void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds, + const u64 ExpectedNumBytes, char *&Buffer) { + char *Ptr = Buffer; + const u64 NumEntries = StackIds.Size(); + Ptr = WriteBytes(NumEntries, Ptr); + for (u64 i = 0; i < NumEntries; i++) { + const u64 Key = StackIds[i]; + MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false); + CHECK(h.exists()); + Ptr = WriteBytes(Key, Ptr); + // FIXME: We unnecessarily serialize the AccessHistogram pointer. Adding a + // serialization schema will fix this issue. See also FIXME in + // deserialization. + Ptr = WriteBytes((*h)->mib, Ptr); + for (u64 j = 0; j < (*h)->mib.AccessHistogramSize; ++j) { + u64 HistogramEntry = ((u64 *)((*h)->mib.AccessHistogram))[j]; + Ptr = WriteBytes(HistogramEntry, Ptr); + } + if ((*h)->mib.AccessHistogramSize > 0) { + InternalFree((void *)((*h)->mib.AccessHistogram)); + } + } + CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && + "Expected num bytes != actual bytes written"); +} + +// Format +// ---------- Header +// Magic +// Version +// Total Size +// Segment Offset +// MIB Info Offset +// Stack Offset +// ---------- Segment Info +// Num Entries +// ---------- Segment Entry +// Start +// End +// Offset +// BuildID 32B +// ---------- +// ... +// ---------- +// Optional Padding Bytes +// ---------- MIB Info +// Num Entries +// ---------- MIB Entry +// Alloc Count +// ... +// ---- AccessHistogram Entry 0 +// ... +// ---- AccessHistogram Entry AccessHistogramSize - 1 +// ---------- MIB Entry 1 +// Alloc Count +// ... +// ---- AccessHistogram Entry 0 +// ... +// ---- AccessHistogram Entry AccessHistogramSize - 1 +// Optional Padding Bytes +// ---------- Stack Info +// Num Entries +// ---------- Stack Entry +// Num Stacks +// PC1 +// PC2 +// ... +// ---------- +// Optional Padding Bytes +// ... +u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules, + char *&Buffer) { + // Each section size is rounded up to 8b since the first entry in each section + // is a u64 which holds the number of entries in the section by convention. + const u64 NumSegmentBytes = RoundUpTo(SegmentSizeBytes(Modules), 8); + + Vector<u64> StackIds; + MIBMap.ForEach(RecordStackId, reinterpret_cast<void *>(&StackIds)); + // The first 8b are for the total number of MIB records. Each MIB record is + // preceded by a 8b stack id which is associated with stack frames in the next + // section. + const u64 NumMIBInfoBytes = RoundUpTo( + sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), 8); + + // Get Number of AccessHistogram entries in total + u64 TotalAccessHistogramEntries = 0; + MIBMap.ForEach( + [](const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, void *Arg) { + u64 *TotalAccessHistogramEntries = (u64 *)Arg; + *TotalAccessHistogramEntries += MIB->mib.AccessHistogramSize; + }, + reinterpret_cast<void *>(&TotalAccessHistogramEntries)); + const u64 NumHistogramBytes = + RoundUpTo(TotalAccessHistogramEntries * sizeof(uint64_t), 8); + + const u64 NumStackBytes = RoundUpTo(StackSizeBytes(StackIds), 8); + + // Ensure that the profile is 8b aligned. We allow for some optional padding + // at the end so that any subsequent profile serialized to the same file does + // not incur unaligned accesses. + const u64 TotalSizeBytes = + RoundUpTo(sizeof(Header) + NumSegmentBytes + NumStackBytes + + NumMIBInfoBytes + NumHistogramBytes, + 8); + + // Allocate the memory for the entire buffer incl. info blocks. + Buffer = (char *)InternalAlloc(TotalSizeBytes); + char *Ptr = Buffer; + + Header header{MEMPROF_RAW_MAGIC_64, + MEMPROF_RAW_VERSION, + static_cast<u64>(TotalSizeBytes), + sizeof(Header), + sizeof(Header) + NumSegmentBytes, + sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes + + NumHistogramBytes}; + Ptr = WriteBytes(header, Ptr); + + SerializeSegmentsToBuffer(Modules, NumSegmentBytes, Ptr); + Ptr += NumSegmentBytes; + + SerializeMIBInfoToBuffer(MIBMap, StackIds, + NumMIBInfoBytes + NumHistogramBytes, Ptr); + Ptr += NumMIBInfoBytes + NumHistogramBytes; + + SerializeStackToBuffer(StackIds, NumStackBytes, Ptr); + + return TotalSizeBytes; +} + +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rawprofile.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rawprofile.h new file mode 100644 index 000000000000..e2494175f165 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rawprofile.h @@ -0,0 +1,15 @@ +#ifndef MEMPROF_RAWPROFILE_H_ +#define MEMPROF_RAWPROFILE_H_ + +#include "memprof_mibmap.h" +#include "sanitizer_common/sanitizer_array_ref.h" +#include "sanitizer_common/sanitizer_common.h" + +namespace __memprof { +// Serialize the in-memory representation of the memprof profile to the raw +// binary format. The format itself is documented memprof_rawprofile.cpp. +u64 SerializeToRawProfile(MIBMapTy &BlockCache, ArrayRef<LoadedModule> Modules, + char *&Buffer); +} // namespace __memprof + +#endif // MEMPROF_RAWPROFILE_H_ diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rtl.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rtl.cpp new file mode 100644 index 000000000000..cf4bde808bfa --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_rtl.cpp @@ -0,0 +1,325 @@ +//===-- memprof_rtl.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Main file of the MemProf run-time library. +//===----------------------------------------------------------------------===// + +#include "memprof_allocator.h" +#include "memprof_interceptors.h" +#include "memprof_interface_internal.h" +#include "memprof_internal.h" +#include "memprof_mapping.h" +#include "memprof_stack.h" +#include "memprof_stats.h" +#include "memprof_thread.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_interface_internal.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_symbolizer.h" + +#include <time.h> + +uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol. + +// Allow the user to specify a profile output file via the binary. +SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1]; + +// Share ClHistogram compiler flag with runtime. +SANITIZER_WEAK_ATTRIBUTE bool __memprof_histogram; + +namespace __memprof { + +static void MemprofDie() { + static atomic_uint32_t num_calls; + if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) { + // Don't die twice - run a busy loop. + while (1) { + internal_sched_yield(); + } + } + if (common_flags()->print_module_map >= 1) + DumpProcessMap(); + if (flags()->unmap_shadow_on_exit) { + if (kHighShadowEnd) + UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg); + } +} + +static void MemprofOnDeadlySignal(int signo, void *siginfo, void *context) { + // We call StartReportDeadlySignal not HandleDeadlySignal so we get the + // deadly signal message to stderr but no writing to the profile output file + StartReportDeadlySignal(); + __memprof_profile_dump(); + Die(); +} + +static void CheckUnwind() { + GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check); + stack.Print(); +} + +// -------------------------- Globals --------------------- {{{1 +int memprof_inited; +bool memprof_init_is_running; +int memprof_timestamp_inited; +long memprof_init_timestamp_s; + +uptr kHighMemEnd; + +// -------------------------- Run-time entry ------------------- {{{1 +// exported functions + +#define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr); +#define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY_HIST() \ + __memprof::RecordAccessHistogram(addr); + +#define MEMPROF_MEMORY_ACCESS_CALLBACK(type) \ + extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) { \ + MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() \ + } + +#define MEMPROF_MEMORY_ACCESS_CALLBACK_HIST(type) \ + extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_hist_##type( \ + uptr addr) { \ + MEMPROF_MEMORY_ACCESS_CALLBACK_BODY_HIST() \ + } + +MEMPROF_MEMORY_ACCESS_CALLBACK_HIST(load) +MEMPROF_MEMORY_ACCESS_CALLBACK_HIST(store) + +MEMPROF_MEMORY_ACCESS_CALLBACK(load) +MEMPROF_MEMORY_ACCESS_CALLBACK(store) + +// Force the linker to keep the symbols for various MemProf interface +// functions. We want to keep those in the executable in order to let the +// instrumented dynamic libraries access the symbol even if it is not used by +// the executable itself. This should help if the build system is removing dead +// code at link time. +static NOINLINE void force_interface_symbols() { + volatile int fake_condition = 0; // prevent dead condition elimination. + // clang-format off + switch (fake_condition) { + case 1: __memprof_record_access(nullptr); break; + case 2: __memprof_record_access_range(nullptr, 0); break; + } + // clang-format on +} + +static void memprof_atexit() { + Printf("MemProfiler exit stats:\n"); + __memprof_print_accumulated_stats(); +} + +static void InitializeHighMemEnd() { + kHighMemEnd = GetMaxUserVirtualAddress(); + // Increase kHighMemEnd to make sure it's properly + // aligned together with kHighMemBeg: + kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1; +} + +void PrintAddressSpaceLayout() { + if (kHighMemBeg) { + Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemBeg, + (void *)kHighMemEnd); + Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg, + (void *)kHighShadowEnd); + } + Printf("|| `[%p, %p]` || ShadowGap ||\n", (void *)kShadowGapBeg, + (void *)kShadowGapEnd); + if (kLowShadowBeg) { + Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowBeg, + (void *)kLowShadowEnd); + Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowMemBeg, + (void *)kLowMemEnd); + } + Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg), + (void *)MEM_TO_SHADOW(kLowShadowEnd)); + if (kHighMemBeg) { + Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg), + (void *)MEM_TO_SHADOW(kHighShadowEnd)); + } + Printf("\n"); + Printf("malloc_context_size=%zu\n", + (uptr)common_flags()->malloc_context_size); + + Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE); + Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY); + Printf("SHADOW_OFFSET: %p\n", (void *)SHADOW_OFFSET); + CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7); +} + +static void MemprofInitInternal() { + if (LIKELY(memprof_inited)) + return; + SanitizerToolName = "MemProfiler"; + CHECK(!memprof_init_is_running && "MemProf init calls itself!"); + memprof_init_is_running = true; + + CacheBinaryName(); + + // Initialize flags. This must be done early, because most of the + // initialization steps look at flags(). + InitializeFlags(); + + AvoidCVE_2016_2143(); + + SetMallocContextSize(common_flags()->malloc_context_size); + + InitializeHighMemEnd(); + + // Make sure we are not statically linked. + __interception::DoesNotSupportStaticLinking(); + + // Install tool-specific callbacks in sanitizer_common. + AddDieCallback(MemprofDie); + SetCheckUnwindCallback(CheckUnwind); + + // Use profile name specified via the binary itself if it exists, and hasn't + // been overrriden by a flag at runtime. + if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path) + __sanitizer_set_report_path(__memprof_profile_filename); + else + __sanitizer_set_report_path(common_flags()->log_path); + + __sanitizer::InitializePlatformEarly(); + + // Setup internal allocator callback. + SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); + + InitializeMemprofInterceptors(); + CheckASLR(); + + ReplaceSystemMalloc(); + + DisableCoreDumperIfNecessary(); + + InitializeShadowMemory(); + + TSDInit(PlatformTSDDtor); + InstallDeadlySignalHandlers(MemprofOnDeadlySignal); + + InitializeAllocator(); + + if (flags()->atexit) + Atexit(memprof_atexit); + + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); + + // interceptors + InitTlsSize(); + + // Create main thread. + MemprofThread *main_thread = CreateMainThread(); + CHECK_EQ(0, main_thread->tid()); + force_interface_symbols(); // no-op. + SanitizerInitializeUnwinder(); + + Symbolizer::LateInitialize(); + + VReport(1, "MemProfiler Init done\n"); + + memprof_init_is_running = false; + memprof_inited = 1; +} + +void MemprofInitTime() { + if (LIKELY(memprof_timestamp_inited)) + return; + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + memprof_init_timestamp_s = ts.tv_sec; + memprof_timestamp_inited = 1; +} + +// Initialize as requested from some part of MemProf runtime library +// (interceptors, allocator, etc). +void MemprofInitFromRtl() { MemprofInitInternal(); } + +#if MEMPROF_DYNAMIC +// Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable +// (and thus normal initializers from .preinit_array or modules haven't run). + +class MemprofInitializer { +public: + MemprofInitializer() { MemprofInitFromRtl(); } +}; + +static MemprofInitializer memprof_initializer; +#endif // MEMPROF_DYNAMIC + +} // namespace __memprof + +// ---------------------- Interface ---------------- {{{1 +using namespace __memprof; + +// Initialize as requested from instrumented application code. +void __memprof_init() { + MemprofInitTime(); + MemprofInitInternal(); +} + +void __memprof_preinit() { MemprofInitInternal(); } + +void __memprof_version_mismatch_check_v1() {} + +void __memprof_record_access(void const volatile *addr) { + __memprof::RecordAccess((uptr)addr); +} + +void __memprof_record_access_hist(void const volatile *addr) { + __memprof::RecordAccessHistogram((uptr)addr); +} + +void __memprof_record_access_range(void const volatile *addr, uptr size) { + for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize) + __memprof::RecordAccess(a); +} + +void __memprof_record_access_range_hist(void const volatile *addr, uptr size) { + for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize) + __memprof::RecordAccessHistogram(a); +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16 +__sanitizer_unaligned_load16(const uu16 *p) { + __memprof_record_access(p); + return *p; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32 +__sanitizer_unaligned_load32(const uu32 *p) { + __memprof_record_access(p); + return *p; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64 +__sanitizer_unaligned_load64(const uu64 *p) { + __memprof_record_access(p); + return *p; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_unaligned_store16(uu16 *p, u16 x) { + __memprof_record_access(p); + *p = x; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_unaligned_store32(uu32 *p, u32 x) { + __memprof_record_access(p); + *p = x; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_unaligned_store64(uu64 *p, u64 x) { + __memprof_record_access(p); + *p = x; +} diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_shadow_setup.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_shadow_setup.cpp new file mode 100644 index 000000000000..e7832f656ee8 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_shadow_setup.cpp @@ -0,0 +1,62 @@ +//===-- memprof_shadow_setup.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Set up the shadow memory. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" + +#include "memprof_internal.h" +#include "memprof_mapping.h" + +namespace __memprof { + +static void ProtectGap(uptr addr, uptr size) { + if (!flags()->protect_shadow_gap) { + // The shadow gap is unprotected, so there is a chance that someone + // is actually using this memory. Which means it needs a shadow... + uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached()); + uptr GapShadowEnd = + RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1; + if (Verbosity()) + Printf("protect_shadow_gap=0:" + " not protecting shadow gap, allocating gap's shadow\n" + "|| `[%p, %p]` || ShadowGap's shadow ||\n", + GapShadowBeg, GapShadowEnd); + ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd, + "unprotected gap shadow"); + return; + } + __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart, + kZeroBaseMaxShadowStart); +} + +void InitializeShadowMemory() { + uptr shadow_start = FindDynamicShadowStart(); + // Update the shadow memory address (potentially) used by instrumentation. + __memprof_shadow_memory_dynamic_address = shadow_start; + + if (kLowShadowBeg) + shadow_start -= GetMmapGranularity(); + + if (Verbosity()) + PrintAddressSpaceLayout(); + + // mmap the low shadow plus at least one page at the left. + if (kLowShadowBeg) + ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); + // mmap the high shadow. + ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); + // protect the gap. + ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); + CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); +} + +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stack.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stack.cpp new file mode 100644 index 000000000000..b5beeeadafd7 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stack.cpp @@ -0,0 +1,59 @@ +//===-- memprof_stack.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Code for MemProf stack trace. +//===----------------------------------------------------------------------===// +#include "memprof_stack.h" +#include "memprof_internal.h" +#include "sanitizer_common/sanitizer_atomic.h" + +namespace __memprof { + +static atomic_uint32_t malloc_context_size; + +void SetMallocContextSize(u32 size) { + atomic_store(&malloc_context_size, size, memory_order_release); +} + +u32 GetMallocContextSize() { + return atomic_load(&malloc_context_size, memory_order_acquire); +} + +} // namespace __memprof + +void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp, + void *context, + bool request_fast, + u32 max_depth) { + using namespace __memprof; + size = 0; + if (UNLIKELY(!memprof_inited)) + return; + request_fast = StackTrace::WillUseFastUnwind(request_fast); + MemprofThread *t = GetCurrentThread(); + if (request_fast) { + if (t) { + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), + true); + } + return; + } + Unwind(max_depth, pc, bp, context, 0, 0, false); +} + +// ------------------ Interface -------------- {{{1 + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_print_stack_trace() { + using namespace __memprof; + PRINT_CURRENT_STACK(); +} +} // extern "C" diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stack.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stack.h new file mode 100644 index 000000000000..a8fdfc9def9d --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stack.h @@ -0,0 +1,66 @@ +//===-- memprof_stack.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header for memprof_stack.cpp. +//===----------------------------------------------------------------------===// + +#ifndef MEMPROF_STACK_H +#define MEMPROF_STACK_H + +#include "memprof_flags.h" +#include "memprof_thread.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_stacktrace.h" + +namespace __memprof { + +static const u32 kDefaultMallocContextSize = 30; + +void SetMallocContextSize(u32 size); +u32 GetMallocContextSize(); + +} // namespace __memprof + +// NOTE: A Rule of thumb is to retrieve stack trace in the interceptors +// as early as possible (in functions exposed to the user), as we generally +// don't want stack trace to contain functions from MemProf internals. + +#define GET_STACK_TRACE(max_size, fast) \ + BufferedStackTrace stack; \ + if (max_size <= 2) { \ + stack.size = max_size; \ + if (max_size > 0) { \ + stack.top_frame_bp = GET_CURRENT_FRAME(); \ + stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ + if (max_size > 1) \ + stack.trace_buffer[1] = GET_CALLER_PC(); \ + } \ + } else { \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ + fast, max_size); \ + } + +#define GET_STACK_TRACE_FATAL_HERE \ + GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) + +#define GET_STACK_TRACE_THREAD GET_STACK_TRACE(kStackTraceMax, true) + +#define GET_STACK_TRACE_MALLOC \ + GET_STACK_TRACE(GetMallocContextSize(), common_flags()->fast_unwind_on_malloc) + +#define GET_STACK_TRACE_FREE GET_STACK_TRACE_MALLOC + +#define PRINT_CURRENT_STACK() \ + { \ + GET_STACK_TRACE_FATAL_HERE; \ + stack.Print(); \ + } + +#endif // MEMPROF_STACK_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stats.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stats.cpp new file mode 100644 index 000000000000..c8faebfa12de --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stats.cpp @@ -0,0 +1,157 @@ +//===-- memprof_stats.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Code related to statistics collected by MemProfiler. +//===----------------------------------------------------------------------===// +#include "memprof_stats.h" +#include "memprof_interceptors.h" +#include "memprof_internal.h" +#include "memprof_thread.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_mutex.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + +namespace __memprof { + +MemprofStats::MemprofStats() { Clear(); } + +void MemprofStats::Clear() { + if (REAL(memset)) + return (void)REAL(memset)(this, 0, sizeof(MemprofStats)); + internal_memset(this, 0, sizeof(MemprofStats)); +} + +static void PrintMallocStatsArray(const char *prefix, + uptr (&array)[kNumberOfSizeClasses]) { + Printf("%s", prefix); + for (uptr i = 0; i < kNumberOfSizeClasses; i++) { + if (!array[i]) + continue; + Printf("%zu:%zu; ", i, array[i]); + } + Printf("\n"); +} + +void MemprofStats::Print() { + Printf("Stats: %zuM malloced (%zuM for overhead) by %zu calls\n", + malloced >> 20, malloced_overhead >> 20, mallocs); + Printf("Stats: %zuM realloced by %zu calls\n", realloced >> 20, reallocs); + Printf("Stats: %zuM freed by %zu calls\n", freed >> 20, frees); + Printf("Stats: %zuM really freed by %zu calls\n", really_freed >> 20, + real_frees); + Printf("Stats: %zuM (%zuM-%zuM) mmaped; %zu maps, %zu unmaps\n", + (mmaped - munmaped) >> 20, mmaped >> 20, munmaped >> 20, mmaps, + munmaps); + + PrintMallocStatsArray(" mallocs by size class: ", malloced_by_size); + Printf("Stats: malloc large: %zu\n", malloc_large); +} + +void MemprofStats::MergeFrom(const MemprofStats *stats) { + uptr *dst_ptr = reinterpret_cast<uptr *>(this); + const uptr *src_ptr = reinterpret_cast<const uptr *>(stats); + uptr num_fields = sizeof(*this) / sizeof(uptr); + for (uptr i = 0; i < num_fields; i++) + dst_ptr[i] += src_ptr[i]; +} + +static Mutex print_lock; + +static MemprofStats unknown_thread_stats(LINKER_INITIALIZED); +static MemprofStats dead_threads_stats(LINKER_INITIALIZED); +static Mutex dead_threads_stats_lock; +// Required for malloc_zone_statistics() on OS X. This can't be stored in +// per-thread MemprofStats. +static uptr max_malloced_memory; + +static void MergeThreadStats(ThreadContextBase *tctx_base, void *arg) { + MemprofStats *accumulated_stats = reinterpret_cast<MemprofStats *>(arg); + MemprofThreadContext *tctx = static_cast<MemprofThreadContext *>(tctx_base); + if (MemprofThread *t = tctx->thread) + accumulated_stats->MergeFrom(&t->stats()); +} + +static void GetAccumulatedStats(MemprofStats *stats) { + stats->Clear(); + { + ThreadRegistryLock l(&memprofThreadRegistry()); + memprofThreadRegistry().RunCallbackForEachThreadLocked(MergeThreadStats, + stats); + } + stats->MergeFrom(&unknown_thread_stats); + { + Lock lock(&dead_threads_stats_lock); + stats->MergeFrom(&dead_threads_stats); + } + // This is not very accurate: we may miss allocation peaks that happen + // between two updates of accumulated_stats_. For more accurate bookkeeping + // the maximum should be updated on every malloc(), which is unacceptable. + if (max_malloced_memory < stats->malloced) { + max_malloced_memory = stats->malloced; + } +} + +void FlushToDeadThreadStats(MemprofStats *stats) { + Lock lock(&dead_threads_stats_lock); + dead_threads_stats.MergeFrom(stats); + stats->Clear(); +} + +MemprofStats &GetCurrentThreadStats() { + MemprofThread *t = GetCurrentThread(); + return (t) ? t->stats() : unknown_thread_stats; +} + +static void PrintAccumulatedStats() { + MemprofStats stats; + GetAccumulatedStats(&stats); + // Use lock to keep reports from mixing up. + Lock lock(&print_lock); + stats.Print(); + StackDepotStats stack_depot_stats = StackDepotGetStats(); + Printf("Stats: StackDepot: %zd ids; %zdM allocated\n", + stack_depot_stats.n_uniq_ids, stack_depot_stats.allocated >> 20); + PrintInternalAllocatorStats(); +} + +} // namespace __memprof + +// ---------------------- Interface ---------------- {{{1 +using namespace __memprof; + +uptr __sanitizer_get_current_allocated_bytes() { + MemprofStats stats; + GetAccumulatedStats(&stats); + uptr malloced = stats.malloced; + uptr freed = stats.freed; + // Return sane value if malloced < freed due to racy + // way we update accumulated stats. + return (malloced > freed) ? malloced - freed : 1; +} + +uptr __sanitizer_get_heap_size() { + MemprofStats stats; + GetAccumulatedStats(&stats); + return stats.mmaped - stats.munmaped; +} + +uptr __sanitizer_get_free_bytes() { + MemprofStats stats; + GetAccumulatedStats(&stats); + uptr total_free = stats.mmaped - stats.munmaped + stats.really_freed; + uptr total_used = stats.malloced; + // Return sane value if total_free < total_used due to racy + // way we update accumulated stats. + return (total_free > total_used) ? total_free - total_used : 1; +} + +uptr __sanitizer_get_unmapped_bytes() { return 0; } + +void __memprof_print_accumulated_stats() { PrintAccumulatedStats(); } diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stats.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stats.h new file mode 100644 index 000000000000..ebdaa1909817 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_stats.h @@ -0,0 +1,61 @@ +//===-- memprof_stats.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header for statistics. +//===----------------------------------------------------------------------===// +#ifndef MEMPROF_STATS_H +#define MEMPROF_STATS_H + +#include "memprof_allocator.h" +#include "memprof_internal.h" + +namespace __memprof { + +// MemprofStats struct is NOT thread-safe. +// Each MemprofThread has its own MemprofStats, which are sometimes flushed +// to the accumulated MemprofStats. +struct MemprofStats { + // MemprofStats must be a struct consisting of uptr fields only. + // When merging two MemprofStats structs, we treat them as arrays of uptr. + uptr mallocs; + uptr malloced; + uptr malloced_overhead; + uptr frees; + uptr freed; + uptr real_frees; + uptr really_freed; + uptr reallocs; + uptr realloced; + uptr mmaps; + uptr mmaped; + uptr munmaps; + uptr munmaped; + uptr malloc_large; + uptr malloced_by_size[kNumberOfSizeClasses]; + + // Ctor for global MemprofStats (accumulated stats for dead threads). + explicit MemprofStats(LinkerInitialized) {} + // Creates empty stats. + MemprofStats(); + + void Print(); // Prints formatted stats to stderr. + void Clear(); + void MergeFrom(const MemprofStats *stats); +}; + +// Returns stats for GetCurrentThread(), or stats for fake "unknown thread" +// if GetCurrentThread() returns 0. +MemprofStats &GetCurrentThreadStats(); +// Flushes a given stats into accumulated stats of dead threads. +void FlushToDeadThreadStats(MemprofStats *stats); + +} // namespace __memprof + +#endif // MEMPROF_STATS_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_thread.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_thread.cpp new file mode 100644 index 000000000000..e2bca9bb422f --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_thread.cpp @@ -0,0 +1,219 @@ +//===-- memprof_thread.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// Thread-related code. +//===----------------------------------------------------------------------===// +#include "memprof_thread.h" +#include "memprof_allocator.h" +#include "memprof_interceptors.h" +#include "memprof_mapping.h" +#include "memprof_stack.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +namespace __memprof { + +// MemprofThreadContext implementation. + +void MemprofThreadContext::OnCreated(void *arg) { + CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs *>(arg); + if (args->stack) + stack_id = StackDepotPut(*args->stack); + thread = args->thread; + thread->set_context(this); +} + +void MemprofThreadContext::OnFinished() { + // Drop the link to the MemprofThread object. + thread = nullptr; +} + +alignas(16) static char thread_registry_placeholder[sizeof(ThreadRegistry)]; +static ThreadRegistry *memprof_thread_registry; + +static Mutex mu_for_thread_context; +static LowLevelAllocator allocator_for_thread_context; + +static ThreadContextBase *GetMemprofThreadContext(u32 tid) { + Lock lock(&mu_for_thread_context); + return new (allocator_for_thread_context) MemprofThreadContext(tid); +} + +ThreadRegistry &memprofThreadRegistry() { + static bool initialized; + // Don't worry about thread_safety - this should be called when there is + // a single thread. + if (!initialized) { + // Never reuse MemProf threads: we store pointer to MemprofThreadContext + // in TSD and can't reliably tell when no more TSD destructors will + // be called. It would be wrong to reuse MemprofThreadContext for another + // thread before all TSD destructors will be called for it. + memprof_thread_registry = new (thread_registry_placeholder) + ThreadRegistry(GetMemprofThreadContext); + initialized = true; + } + return *memprof_thread_registry; +} + +MemprofThreadContext *GetThreadContextByTidLocked(u32 tid) { + return static_cast<MemprofThreadContext *>( + memprofThreadRegistry().GetThreadLocked(tid)); +} + +// MemprofThread implementation. + +MemprofThread *MemprofThread::Create(thread_callback_t start_routine, void *arg, + u32 parent_tid, StackTrace *stack, + bool detached) { + uptr PageSize = GetPageSizeCached(); + uptr size = RoundUpTo(sizeof(MemprofThread), PageSize); + MemprofThread *thread = (MemprofThread *)MmapOrDie(size, __func__); + thread->start_routine_ = start_routine; + thread->arg_ = arg; + MemprofThreadContext::CreateThreadContextArgs args = {thread, stack}; + memprofThreadRegistry().CreateThread(0, detached, parent_tid, &args); + + return thread; +} + +void MemprofThread::TSDDtor(void *tsd) { + MemprofThreadContext *context = (MemprofThreadContext *)tsd; + VReport(1, "T%d TSDDtor\n", context->tid); + if (context->thread) + context->thread->Destroy(); +} + +void MemprofThread::Destroy() { + int tid = this->tid(); + VReport(1, "T%d exited\n", tid); + + malloc_storage().CommitBack(); + memprofThreadRegistry().FinishThread(tid); + FlushToDeadThreadStats(&stats_); + uptr size = RoundUpTo(sizeof(MemprofThread), GetPageSizeCached()); + UnmapOrDie(this, size); + DTLS_Destroy(); +} + +inline MemprofThread::StackBounds MemprofThread::GetStackBounds() const { + if (stack_bottom_ >= stack_top_) + return {0, 0}; + return {stack_bottom_, stack_top_}; +} + +uptr MemprofThread::stack_top() { return GetStackBounds().top; } + +uptr MemprofThread::stack_bottom() { return GetStackBounds().bottom; } + +uptr MemprofThread::stack_size() { + const auto bounds = GetStackBounds(); + return bounds.top - bounds.bottom; +} + +void MemprofThread::Init(const InitOptions *options) { + CHECK_EQ(this->stack_size(), 0U); + SetThreadStackAndTls(options); + if (stack_top_ != stack_bottom_) { + CHECK_GT(this->stack_size(), 0U); + CHECK(AddrIsInMem(stack_bottom_)); + CHECK(AddrIsInMem(stack_top_ - 1)); + } + int local = 0; + VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(), + (void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_, + (void *)&local); +} + +thread_return_t +MemprofThread::ThreadStart(tid_t os_id, + atomic_uintptr_t *signal_thread_is_registered) { + Init(); + memprofThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, + nullptr); + if (signal_thread_is_registered) + atomic_store(signal_thread_is_registered, 1, memory_order_release); + + if (!start_routine_) { + // start_routine_ == 0 if we're on the main thread or on one of the + // OS X libdispatch worker threads. But nobody is supposed to call + // ThreadStart() for the worker threads. + CHECK_EQ(tid(), 0); + return 0; + } + + return start_routine_(arg_); +} + +MemprofThread *CreateMainThread() { + MemprofThread *main_thread = MemprofThread::Create( + /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ kMainTid, + /* stack */ nullptr, /* detached */ true); + SetCurrentThread(main_thread); + main_thread->ThreadStart(internal_getpid(), + /* signal_thread_is_registered */ nullptr); + return main_thread; +} + +// This implementation doesn't use the argument, which is just passed down +// from the caller of Init (which see, above). It's only there to support +// OS-specific implementations that need more information passed through. +void MemprofThread::SetThreadStackAndTls(const InitOptions *options) { + DCHECK_EQ(options, nullptr); + uptr tls_size = 0; + uptr stack_size = 0; + GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size, + &tls_begin_, &tls_size); + stack_top_ = stack_bottom_ + stack_size; + tls_end_ = tls_begin_ + tls_size; + dtls_ = DTLS_Get(); + + if (stack_top_ != stack_bottom_) { + int local; + CHECK(AddrIsInStack((uptr)&local)); + } +} + +bool MemprofThread::AddrIsInStack(uptr addr) { + const auto bounds = GetStackBounds(); + return addr >= bounds.bottom && addr < bounds.top; +} + +MemprofThread *GetCurrentThread() { + MemprofThreadContext *context = + reinterpret_cast<MemprofThreadContext *>(TSDGet()); + if (!context) + return nullptr; + return context->thread; +} + +void SetCurrentThread(MemprofThread *t) { + CHECK(t->context()); + VReport(2, "SetCurrentThread: %p for thread %p\n", (void *)t->context(), + (void *)GetThreadSelf()); + // Make sure we do not reset the current MemprofThread. + CHECK_EQ(0, TSDGet()); + TSDSet(t->context()); + CHECK_EQ(t->context(), TSDGet()); +} + +u32 GetCurrentTidOrInvalid() { + MemprofThread *t = GetCurrentThread(); + return t ? t->tid() : kInvalidTid; +} + +void EnsureMainThreadIDIsCorrect() { + MemprofThreadContext *context = + reinterpret_cast<MemprofThreadContext *>(TSDGet()); + if (context && (context->tid == kMainTid)) + context->os_id = GetTid(); +} +} // namespace __memprof diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/memprof_thread.h b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_thread.h new file mode 100644 index 000000000000..4c9313fcb369 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/memprof_thread.h @@ -0,0 +1,135 @@ +//===-- memprof_thread.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemProfiler, a memory profiler. +// +// MemProf-private header for memprof_thread.cpp. +//===----------------------------------------------------------------------===// + +#ifndef MEMPROF_THREAD_H +#define MEMPROF_THREAD_H + +#include "memprof_allocator.h" +#include "memprof_internal.h" +#include "memprof_stats.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_thread_registry.h" + +namespace __sanitizer { +struct DTLS; +} // namespace __sanitizer + +namespace __memprof { + +class MemprofThread; + +// These objects are created for every thread and are never deleted, +// so we can find them by tid even if the thread is long dead. +struct MemprofThreadContext final : public ThreadContextBase { + explicit MemprofThreadContext(int tid) + : ThreadContextBase(tid), announced(false), + destructor_iterations(GetPthreadDestructorIterations()), stack_id(0), + thread(nullptr) {} + bool announced; + u8 destructor_iterations; + u32 stack_id; + MemprofThread *thread; + + void OnCreated(void *arg) override; + void OnFinished() override; + + struct CreateThreadContextArgs { + MemprofThread *thread; + StackTrace *stack; + }; +}; + +// MemprofThreadContext objects are never freed, so we need many of them. +COMPILER_CHECK(sizeof(MemprofThreadContext) <= 256); + +// MemprofThread are stored in TSD and destroyed when the thread dies. +class MemprofThread { +public: + static MemprofThread *Create(thread_callback_t start_routine, void *arg, + u32 parent_tid, StackTrace *stack, + bool detached); + static void TSDDtor(void *tsd); + void Destroy(); + + struct InitOptions; + void Init(const InitOptions *options = nullptr); + + thread_return_t ThreadStart(tid_t os_id, + atomic_uintptr_t *signal_thread_is_registered); + + uptr stack_top(); + uptr stack_bottom(); + uptr stack_size(); + uptr tls_begin() { return tls_begin_; } + uptr tls_end() { return tls_end_; } + DTLS *dtls() { return dtls_; } + u32 tid() { return context_->tid; } + MemprofThreadContext *context() { return context_; } + void set_context(MemprofThreadContext *context) { context_ = context; } + + bool AddrIsInStack(uptr addr); + + // True is this thread is currently unwinding stack (i.e. collecting a stack + // trace). Used to prevent deadlocks on platforms where libc unwinder calls + // malloc internally. See PR17116 for more details. + bool isUnwinding() const { return unwinding_; } + void setUnwinding(bool b) { unwinding_ = b; } + + MemprofThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } + MemprofStats &stats() { return stats_; } + +private: + // NOTE: There is no MemprofThread constructor. It is allocated + // via mmap() and *must* be valid in zero-initialized state. + + void SetThreadStackAndTls(const InitOptions *options); + + struct StackBounds { + uptr bottom; + uptr top; + }; + StackBounds GetStackBounds() const; + + MemprofThreadContext *context_; + thread_callback_t start_routine_; + void *arg_; + + uptr stack_top_; + uptr stack_bottom_; + + uptr tls_begin_; + uptr tls_end_; + DTLS *dtls_; + + MemprofThreadLocalMallocStorage malloc_storage_; + MemprofStats stats_; + bool unwinding_; +}; + +// Returns a single instance of registry. +ThreadRegistry &memprofThreadRegistry(); + +// Must be called under ThreadRegistryLock. +MemprofThreadContext *GetThreadContextByTidLocked(u32 tid); + +// Get the current thread. May return 0. +MemprofThread *GetCurrentThread(); +void SetCurrentThread(MemprofThread *t); +u32 GetCurrentTidOrInvalid(); + +// Used to handle fork(). +void EnsureMainThreadIDIsCorrect(); +} // namespace __memprof + +#endif // MEMPROF_THREAD_H diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/tests/driver.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/tests/driver.cpp new file mode 100644 index 000000000000..b402cec1126b --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/tests/driver.cpp @@ -0,0 +1,14 @@ +//===-- driver.cpp ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gtest/gtest.h" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/tests/rawprofile.cpp b/contrib/llvm-project/compiler-rt/lib/memprof/tests/rawprofile.cpp new file mode 100644 index 000000000000..5764af9ce8af --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/tests/rawprofile.cpp @@ -0,0 +1,170 @@ +#include "memprof/memprof_rawprofile.h" + +#include <cstdint> +#include <memory> + +#include "profile/MemProfData.inc" +#include "sanitizer_common/sanitizer_array_ref.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace { + +using ::__memprof::MIBMapTy; +using ::__memprof::SerializeToRawProfile; +using ::__sanitizer::StackDepotPut; +using ::__sanitizer::StackTrace; +using ::llvm::memprof::MemInfoBlock; + +uint64_t PopulateFakeMap(const MemInfoBlock &FakeMIB, uintptr_t StackPCBegin, + MIBMapTy &FakeMap) { + constexpr int kSize = 5; + uintptr_t array[kSize]; + for (int i = 0; i < kSize; i++) { + array[i] = StackPCBegin + i; + } + StackTrace St(array, kSize); + uint32_t Id = StackDepotPut(St); + + InsertOrMerge(Id, FakeMIB, FakeMap); + return Id; +} + +template <class T = uint64_t> T Read(char *&Buffer) { + static_assert(std::is_pod<T>::value, "Must be a POD type."); + assert(reinterpret_cast<size_t>(Buffer) % sizeof(T) == 0 && + "Unaligned read!"); + T t = *reinterpret_cast<T *>(Buffer); + Buffer += sizeof(T); + return t; +} + +TEST(MemProf, Basic) { + __sanitizer::LoadedModule FakeModule; + FakeModule.addAddressRange(/*begin=*/0x10, /*end=*/0x20, /*executable=*/true, + /*writable=*/false, /*name=*/""); + const char uuid[MEMPROF_BUILDID_MAX_SIZE] = {0xC, 0x0, 0xF, 0xF, 0xE, 0xE}; + FakeModule.setUuid(uuid, MEMPROF_BUILDID_MAX_SIZE); + __sanitizer::ArrayRef<__sanitizer::LoadedModule> Modules(&FakeModule, + (&FakeModule) + 1); + + MIBMapTy FakeMap; + MemInfoBlock FakeMIB; + // Since we want to override the constructor set vals to make it easier to + // test. + memset(&FakeMIB, 0, sizeof(MemInfoBlock)); + FakeMIB.AllocCount = 0x1; + FakeMIB.TotalAccessCount = 0x2; + + uint64_t FakeIds[2]; + FakeIds[0] = PopulateFakeMap(FakeMIB, /*StackPCBegin=*/2, FakeMap); + FakeIds[1] = PopulateFakeMap(FakeMIB, /*StackPCBegin=*/3, FakeMap); + + char *Ptr = nullptr; + uint64_t NumBytes = SerializeToRawProfile(FakeMap, Modules, Ptr); + const char *Buffer = Ptr; + + ASSERT_GT(NumBytes, 0ULL); + ASSERT_TRUE(Ptr); + + // Check the header. + EXPECT_THAT(Read(Ptr), MEMPROF_RAW_MAGIC_64); + EXPECT_THAT(Read(Ptr), MEMPROF_RAW_VERSION); + const uint64_t TotalSize = Read(Ptr); + const uint64_t SegmentOffset = Read(Ptr); + const uint64_t MIBOffset = Read(Ptr); + const uint64_t StackOffset = Read(Ptr); + + // ============= Check sizes and padding. + EXPECT_EQ(TotalSize, NumBytes); + EXPECT_EQ(TotalSize % 8, 0ULL); + + // Should be equal to the size of the raw profile header. + EXPECT_EQ(SegmentOffset, 48ULL); + + // We expect only 1 segment entry, 8b for the count and 64b for SegmentEntry + // in memprof_rawprofile.cpp. + EXPECT_EQ(MIBOffset - SegmentOffset, 72ULL); + + EXPECT_EQ(MIBOffset, 120ULL); + // We expect 2 mib entry, 8b for the count and sizeof(uint64_t) + + // sizeof(MemInfoBlock) contains stack id + MeminfoBlock. + EXPECT_EQ(StackOffset - MIBOffset, 8 + 2 * (8 + sizeof(MemInfoBlock))); + + EXPECT_EQ(StackOffset, 432ULL); + // We expect 2 stack entries, with 5 frames - 8b for total count, + // 2 * (8b for id, 8b for frame count and 5*8b for fake frames). + // Since this is the last section, there may be additional padding at the end + // to make the total profile size 8b aligned. + EXPECT_GE(TotalSize - StackOffset, 8ULL + 2 * (8 + 8 + 5 * 8)); + + // ============= Check contents. + unsigned char ExpectedSegmentBytes[72] = { + 0x01, 0, 0, 0, 0, 0, 0, 0, // Number of entries + 0x10, 0, 0, 0, 0, 0, 0, 0, // Start + 0x20, 0, 0, 0, 0, 0, 0, 0, // End + 0x0, 0, 0, 0, 0, 0, 0, 0, // Offset + 0x20, 0, 0, 0, 0, 0, 0, 0, // UuidSize + 0xC, 0x0, 0xF, 0xF, 0xE, 0xE // Uuid + }; + EXPECT_EQ(memcmp(Buffer + SegmentOffset, ExpectedSegmentBytes, 72), 0); + + // Check that the number of entries is 2. + EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + MIBOffset), 2ULL); + // Check that stack id is set. + EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + MIBOffset + 8), + FakeIds[0]); + + // Only check a few fields of the first MemInfoBlock. + unsigned char ExpectedMIBBytes[sizeof(MemInfoBlock)] = { + 0x01, 0, 0, 0, // Alloc count + 0x02, 0, 0, 0, // Total access count + }; + // Compare contents of 1st MIB after skipping count and stack id. + EXPECT_EQ( + memcmp(Buffer + MIBOffset + 16, ExpectedMIBBytes, sizeof(MemInfoBlock)), + 0); + // Compare contents of 2nd MIB after skipping count and stack id for the first + // and only the id for the second. + EXPECT_EQ(memcmp(Buffer + MIBOffset + 16 + sizeof(MemInfoBlock) + 8, + ExpectedMIBBytes, sizeof(MemInfoBlock)), + 0); + + // Check that the number of entries is 2. + EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + StackOffset), 2ULL); + // Check that the 1st stack id is set. + EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + StackOffset + 8), + FakeIds[0]); + // Contents are num pcs, value of each pc - 1. + unsigned char ExpectedStackBytes[2][6 * 8] = { + { + 0x5, 0, 0, 0, 0, 0, 0, 0, // Number of PCs + 0x1, 0, 0, 0, 0, 0, 0, 0, // PC ... + 0x2, 0, 0, 0, 0, 0, 0, 0, 0x3, 0, 0, 0, 0, 0, 0, 0, + 0x4, 0, 0, 0, 0, 0, 0, 0, 0x5, 0, 0, 0, 0, 0, 0, 0, + }, + { + 0x5, 0, 0, 0, 0, 0, 0, 0, // Number of PCs + 0x2, 0, 0, 0, 0, 0, 0, 0, // PC ... + 0x3, 0, 0, 0, 0, 0, 0, 0, 0x4, 0, 0, 0, 0, 0, 0, 0, + 0x5, 0, 0, 0, 0, 0, 0, 0, 0x6, 0, 0, 0, 0, 0, 0, 0, + }, + }; + EXPECT_EQ(memcmp(Buffer + StackOffset + 16, ExpectedStackBytes[0], + sizeof(ExpectedStackBytes[0])), + 0); + + // Check that the 2nd stack id is set. + EXPECT_EQ( + *reinterpret_cast<const uint64_t *>(Buffer + StackOffset + 8 + 6 * 8 + 8), + FakeIds[1]); + + EXPECT_EQ(memcmp(Buffer + StackOffset + 16 + 6 * 8 + 8, ExpectedStackBytes[1], + sizeof(ExpectedStackBytes[1])), + 0); +} +} // namespace diff --git a/contrib/llvm-project/compiler-rt/lib/memprof/weak_symbols.txt b/contrib/llvm-project/compiler-rt/lib/memprof/weak_symbols.txt new file mode 100644 index 000000000000..271813612ab6 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/memprof/weak_symbols.txt @@ -0,0 +1 @@ +___memprof_default_options __memprof_profile_filename |