diff options
Diffstat (limited to 'compiler-rt')
33 files changed, 841 insertions, 410 deletions
diff --git a/compiler-rt/include/profile/MemProfData.inc b/compiler-rt/include/profile/MemProfData.inc new file mode 100644 index 000000000000..d64227e4ba31 --- /dev/null +++ b/compiler-rt/include/profile/MemProfData.inc @@ -0,0 +1,61 @@ +#ifndef MEMPROF_DATA_INC +#define MEMPROF_DATA_INC +/*===-- MemProfData.inc - MemProf profiling runtime structures -*- C++ -*-=== *\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This is the main file that defines all the data structure, signature, + * constant literals that are shared across profiling runtime library, + * and host tools (reader/writer). + * + * This file has two identical copies. The primary copy lives in LLVM and + * the other one sits in compiler-rt/include/profile directory. To make changes + * in this file, first modify the primary copy and copy it over to compiler-rt. + * Testing of any change in this file can start only after the two copies are + * synced up. + * +\*===----------------------------------------------------------------------===*/ + + +#ifdef _MSC_VER +#define PACKED(__decl__) __pragma(pack(push,1)) __decl__ __pragma(pack(pop)) +#else +#define PACKED(__decl__) __decl__ __attribute__((__packed__)) +#endif + +// A 64-bit magic number to uniquely identify the raw binary memprof profile file. +#define MEMPROF_RAW_MAGIC_64 \ + ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | \ + (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129) + +// The version number of the raw binary format. +#define MEMPROF_RAW_VERSION 1ULL + +namespace llvm { +namespace memprof { +// A struct describing the header used for the raw binary memprof profile format. +PACKED(struct Header { + uint64_t Magic; + uint64_t Version; + uint64_t TotalSize; + uint64_t SegmentOffset; + uint64_t MIBOffset; + uint64_t StackOffset; +}); + +// A struct describing the information necessary to describe a /proc/maps +// segment entry for a particular binary/library identified by its build id. +PACKED(struct SegmentEntry { + uint64_t Start; + uint64_t End; + uint64_t Offset; + uint8_t BuildId[32]; +}); +} // namespace memprof +} // namespace llvm + +#endif diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp index b28909152e20..2ff314a5a9cb 100644 --- a/compiler-rt/lib/asan/asan_interceptors.cpp +++ b/compiler-rt/lib/asan/asan_interceptors.cpp @@ -130,23 +130,24 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) // Strict init-order checking is dlopen-hostile: // https://github.com/google/sanitizers/issues/178 -#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \ - do { \ - if (flags()->strict_init_order) \ - StopInitOrderChecking(); \ - CheckNoDeepBind(filename, flag); \ - } while (false) -#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) -#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() -#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!asan_inited) -#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ - if (AsanThread *t = GetCurrentThread()) { \ - *begin = t->tls_begin(); \ - *end = t->tls_end(); \ - } else { \ - *begin = *end = 0; \ - } +# define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ + ({ \ + if (flags()->strict_init_order) \ + StopInitOrderChecking(); \ + CheckNoDeepBind(filename, flag); \ + REAL(dlopen)(filename, flag); \ + }) +# define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() +# define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) +# define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() +# define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!asan_inited) +# define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ + if (AsanThread *t = GetCurrentThread()) { \ + *begin = t->tls_begin(); \ + *end = t->tls_end(); \ + } else { \ + *begin = *end = 0; \ + } #define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \ do { \ diff --git a/compiler-rt/lib/asan/asan_report.cpp b/compiler-rt/lib/asan/asan_report.cpp index 1f266334b311..2a38fabaf220 100644 --- a/compiler-rt/lib/asan/asan_report.cpp +++ b/compiler-rt/lib/asan/asan_report.cpp @@ -460,6 +460,10 @@ static bool SuppressErrorReport(uptr pc) { void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write, uptr access_size, u32 exp, bool fatal) { + if (__asan_test_only_reported_buggy_pointer) { + *__asan_test_only_reported_buggy_pointer = addr; + return; + } if (!fatal && SuppressErrorReport(pc)) return; ENABLE_FRAME_POINTER; diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp index 1b150b393cfe..5be8ef0f6d1c 100644 --- a/compiler-rt/lib/asan/asan_rtl.cpp +++ b/compiler-rt/lib/asan/asan_rtl.cpp @@ -85,12 +85,8 @@ void ShowStatsAndAbort() { NOINLINE static void ReportGenericErrorWrapper(uptr addr, bool is_write, int size, int exp_arg, bool fatal) { - if (__asan_test_only_reported_buggy_pointer) { - *__asan_test_only_reported_buggy_pointer = addr; - } else { - GET_CALLER_PC_BP_SP; - ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, fatal); - } + GET_CALLER_PC_BP_SP; + ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, fatal); } // --------------- LowLevelAllocateCallbac ---------- {{{1 diff --git a/compiler-rt/lib/cfi/cfi.cpp b/compiler-rt/lib/cfi/cfi.cpp index 95853208f951..65a10c999cc6 100644 --- a/compiler-rt/lib/cfi/cfi.cpp +++ b/compiler-rt/lib/cfi/cfi.cpp @@ -230,7 +230,7 @@ uptr find_cfi_check_in_dso(dl_phdr_info *info) { } if (symtab > strtab) { - VReport(1, "Can not handle: symtab > strtab (%p > %zx)\n", symtab, strtab); + VReport(1, "Can not handle: symtab > strtab (%zx > %zx)\n", symtab, strtab); return 0; } @@ -250,7 +250,7 @@ uptr find_cfi_check_in_dso(dl_phdr_info *info) { if (phdr_idx == info->dlpi_phnum) { // Nope, either different segments or just bogus pointers. // Can not handle this. - VReport(1, "Can not handle: symtab %p, strtab %zx\n", symtab, strtab); + VReport(1, "Can not handle: symtab %zx, strtab %zx\n", symtab, strtab); return 0; } diff --git a/compiler-rt/lib/memprof/memprof_allocator.cpp b/compiler-rt/lib/memprof/memprof_allocator.cpp index 696f64d8c324..059ce283b8c9 100644 --- a/compiler-rt/lib/memprof/memprof_allocator.cpp +++ b/compiler-rt/lib/memprof/memprof_allocator.cpp @@ -252,6 +252,8 @@ struct Allocator { InsertLiveBlocks(); if (print_text) { + if (!flags()->print_terse) + Printf("Recorded MIBs (incl. live on exit):\n"); MIBMap.ForEach(PrintCallback, reinterpret_cast<void *>(flags()->print_terse)); StackDepotPrintAll(); @@ -271,9 +273,6 @@ struct Allocator { // Inserts any blocks which have been allocated but not yet deallocated. void InsertLiveBlocks() { - if (print_text && !flags()->print_terse) - Printf("Live on exit:\n"); - allocator.ForEachChunk( [](uptr chunk, void *alloc) { u64 user_requested_size; diff --git a/compiler-rt/lib/memprof/memprof_interceptors.cpp b/compiler-rt/lib/memprof/memprof_interceptors.cpp index 5575ae2fe444..459ad03e8dfe 100644 --- a/compiler-rt/lib/memprof/memprof_interceptors.cpp +++ b/compiler-rt/lib/memprof/memprof_interceptors.cpp @@ -93,10 +93,6 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) do { \ } while (false) #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) -#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \ - do { \ - CheckNoDeepBind(filename, flag); \ - } while (false) #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() diff --git a/compiler-rt/lib/memprof/memprof_rawprofile.cpp b/compiler-rt/lib/memprof/memprof_rawprofile.cpp index 96f315f95b24..c4800a6df34c 100644 --- a/compiler-rt/lib/memprof/memprof_rawprofile.cpp +++ b/compiler-rt/lib/memprof/memprof_rawprofile.cpp @@ -1,6 +1,12 @@ -#include "memprof_rawprofile.h" +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + #include "memprof_meminfoblock.h" +#include "memprof_rawprofile.h" +#include "profile/MemProfData.inc" #include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_procmaps.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -8,29 +14,12 @@ #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_vector.h" -#include <stdlib.h> -#include <string.h> - namespace __memprof { using ::__sanitizer::Vector; +using SegmentEntry = ::llvm::memprof::SegmentEntry; +using Header = ::llvm::memprof::Header; namespace { -typedef struct __attribute__((__packed__)) { - u64 start; - u64 end; - u64 offset; - u8 buildId[32]; -} SegmentEntry; - -typedef struct __attribute__((__packed__)) { - u64 magic; - u64 version; - u64 total_size; - u64 segment_offset; - u64 mib_offset; - u64 stack_offset; -} Header; - template <class T> char *WriteBytes(T Pod, char *&Buffer) { *(T *)Buffer = Pod; return Buffer + sizeof(T); @@ -76,12 +65,12 @@ void SerializeSegmentsToBuffer(MemoryMappingLayoutBase &Layout, for (Layout.Reset(); Layout.Next(&segment);) { if (segment.IsReadable() && segment.IsExecutable()) { - SegmentEntry entry{}; - entry.start = segment.start; - entry.end = segment.end; - entry.offset = segment.offset; - memcpy(entry.buildId, segment.uuid, sizeof(segment.uuid)); - memcpy(Ptr, &entry, sizeof(SegmentEntry)); + SegmentEntry Entry{}; + Entry.Start = segment.start; + Entry.End = segment.end; + Entry.Offset = segment.offset; + memcpy(Entry.BuildId, segment.uuid, sizeof(segment.uuid)); + memcpy(Ptr, &Entry, sizeof(SegmentEntry)); Ptr += sizeof(SegmentEntry); NumSegmentsRecorded++; } @@ -89,7 +78,7 @@ void SerializeSegmentsToBuffer(MemoryMappingLayoutBase &Layout, // Store the number of segments we recorded in the space we reserved. *((u64 *)Buffer) = NumSegmentsRecorded; - CHECK(ExpectedNumBytes == static_cast<u64>(Ptr - Buffer) && + CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && "Expected num bytes != actual bytes written"); } @@ -144,7 +133,7 @@ void SerializeStackToBuffer(const Vector<u64> &StackIds, *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count; } - CHECK(ExpectedNumBytes == static_cast<u64>(Ptr - Buffer) && + CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && "Expected num bytes != actual bytes written"); } @@ -172,7 +161,7 @@ void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds, Ptr = WriteBytes((*h)->mib, Ptr); } - CHECK(ExpectedNumBytes == static_cast<u64>(Ptr - Buffer) && + CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && "Expected num bytes != actual bytes written"); } @@ -193,11 +182,15 @@ void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds, // BuildID 32B // ---------- // ... +// ---------- +// Optional Padding Bytes // ---------- MIB Info // Num Entries // ---------- MIB Entry // Alloc Count // ... +// ---------- +// Optional Padding Bytes // ---------- Stack Info // Num Entries // ---------- Stack Entry @@ -206,23 +199,29 @@ void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds, // PC2 // ... // ---------- +// Optional Padding Bytes // ... u64 SerializeToRawProfile(MIBMapTy &MIBMap, MemoryMappingLayoutBase &Layout, char *&Buffer) { - const u64 NumSegmentBytes = SegmentSizeBytes(Layout); + // Each section size is rounded up to 8b since the first entry in each section + // is a u64 which holds the number of entries in the section by convention. + const u64 NumSegmentBytes = RoundUpTo(SegmentSizeBytes(Layout), 8); Vector<u64> StackIds; MIBMap.ForEach(RecordStackId, reinterpret_cast<void *>(&StackIds)); // The first 8b are for the total number of MIB records. Each MIB record is // preceded by a 8b stack id which is associated with stack frames in the next // section. - const u64 NumMIBInfoBytes = - sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)); + const u64 NumMIBInfoBytes = RoundUpTo( + sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), 8); - const u64 NumStackBytes = StackSizeBytes(StackIds); + const u64 NumStackBytes = RoundUpTo(StackSizeBytes(StackIds), 8); - const u64 TotalSizeBytes = - sizeof(Header) + NumSegmentBytes + NumStackBytes + NumMIBInfoBytes; + // Ensure that the profile is 8b aligned. We allow for some optional padding + // at the end so that any subsequent profile serialized to the same file does + // not incur unaligned accesses. + const u64 TotalSizeBytes = RoundUpTo( + sizeof(Header) + NumSegmentBytes + NumStackBytes + NumMIBInfoBytes, 8); // Allocate the memory for the entire buffer incl. info blocks. Buffer = (char *)InternalAlloc(TotalSizeBytes); diff --git a/compiler-rt/lib/memprof/memprof_rawprofile.h b/compiler-rt/lib/memprof/memprof_rawprofile.h index 052bac3267f1..575104e7e34e 100644 --- a/compiler-rt/lib/memprof/memprof_rawprofile.h +++ b/compiler-rt/lib/memprof/memprof_rawprofile.h @@ -5,17 +5,10 @@ #include "sanitizer_common/sanitizer_procmaps.h" namespace __memprof { - -// TODO: pull these in from MemProfData.inc -#define MEMPROF_RAW_MAGIC_64 \ - (u64)255 << 56 | (u64)'m' << 48 | (u64)'p' << 40 | (u64)'r' << 32 | \ - (u64)'o' << 24 | (u64)'f' << 16 | (u64)'r' << 8 | (u64)129 - -#define MEMPROF_RAW_VERSION 1ULL - +// Serialize the in-memory representation of the memprof profile to the raw +// binary format. The format itself is documented memprof_rawprofile.cpp. u64 SerializeToRawProfile(MIBMapTy &BlockCache, MemoryMappingLayoutBase &Layout, char *&Buffer); - } // namespace __memprof #endif // MEMPROF_RAWPROFILE_H_ diff --git a/compiler-rt/lib/memprof/tests/rawprofile.cpp b/compiler-rt/lib/memprof/tests/rawprofile.cpp index 4404ab86092e..829e18370737 100644 --- a/compiler-rt/lib/memprof/tests/rawprofile.cpp +++ b/compiler-rt/lib/memprof/tests/rawprofile.cpp @@ -1,6 +1,10 @@ #include "memprof/memprof_rawprofile.h" +#include <cstdint> +#include <memory> + #include "memprof/memprof_meminfoblock.h" +#include "profile/MemProfData.inc" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_procmaps.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -8,8 +12,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include <memory> - namespace { using ::__memprof::MemInfoBlock; @@ -47,6 +49,8 @@ u64 PopulateFakeMap(const MemInfoBlock &FakeMIB, uptr StackPCBegin, template <class T = u64> T Read(char *&Buffer) { static_assert(std::is_pod<T>::value, "Must be a POD type."); + assert(reinterpret_cast<size_t>(Buffer) % sizeof(T) == 0 && + "Unaligned read!"); T t = *reinterpret_cast<T *>(Buffer); Buffer += sizeof(T); return t; @@ -101,8 +105,9 @@ TEST(MemProf, Basic) { const u64 MIBOffset = Read(Ptr); const u64 StackOffset = Read(Ptr); - // ============= Check sizes. + // ============= Check sizes and padding. EXPECT_EQ(TotalSize, NumBytes); + EXPECT_EQ(TotalSize % 8, 0ULL); // Should be equal to the size of the raw profile header. EXPECT_EQ(SegmentOffset, 48ULL); @@ -118,8 +123,10 @@ TEST(MemProf, Basic) { EXPECT_EQ(StackOffset, 336ULL); // We expect 2 stack entries, with 5 frames - 8b for total count, - // 2 * (8b for id, 8b for frame count and 5*8b for fake frames) - EXPECT_EQ(TotalSize - StackOffset, 8ULL + 2 * (8 + 8 + 5 * 8)); + // 2 * (8b for id, 8b for frame count and 5*8b for fake frames). + // Since this is the last section, there may be additional padding at the end + // to make the total profile size 8b aligned. + EXPECT_GE(TotalSize - StackOffset, 8ULL + 2 * (8 + 8 + 5 * 8)); // ============= Check contents. unsigned char ExpectedSegmentBytes[64] = { diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 674b1898b046..80db2527461e 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -34,7 +34,8 @@ uint64_t lprofGetLoadModuleSignature() { const __llvm_profile_data *FirstD = __llvm_profile_begin_data(); return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) + - (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0) + Version; + (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0) + Version + + __llvm_profile_get_magic(); } /* Returns 1 if profile is not structurally compatible. */ diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp index bcb7370a7906..af0b0949a88e 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp @@ -24,66 +24,6 @@ namespace __sanitizer { const char *PrimaryAllocatorName = "SizeClassAllocator"; const char *SecondaryAllocatorName = "LargeMmapAllocator"; -// ThreadSanitizer for Go uses libc malloc/free. -#if defined(SANITIZER_USE_MALLOC) -# if SANITIZER_LINUX && !SANITIZER_ANDROID -extern "C" void *__libc_malloc(uptr size); -# if !SANITIZER_GO -extern "C" void *__libc_memalign(uptr alignment, uptr size); -# endif -extern "C" void *__libc_realloc(void *ptr, uptr size); -extern "C" void __libc_free(void *ptr); -# else -# include <stdlib.h> -# define __libc_malloc malloc -# if !SANITIZER_GO -static void *__libc_memalign(uptr alignment, uptr size) { - void *p; - uptr error = posix_memalign(&p, alignment, size); - if (error) return nullptr; - return p; -} -# endif -# define __libc_realloc realloc -# define __libc_free free -# endif - -static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, - uptr alignment) { - (void)cache; -#if !SANITIZER_GO - if (alignment == 0) - return __libc_malloc(size); - else - return __libc_memalign(alignment, size); -#else - // Windows does not provide __libc_memalign/posix_memalign. It provides - // __aligned_malloc, but the allocated blocks can't be passed to free, - // they need to be passed to __aligned_free. InternalAlloc interface does - // not account for such requirement. Alignemnt does not seem to be used - // anywhere in runtime, so just call __libc_malloc for now. - DCHECK_EQ(alignment, 0); - return __libc_malloc(size); -#endif -} - -static void *RawInternalRealloc(void *ptr, uptr size, - InternalAllocatorCache *cache) { - (void)cache; - return __libc_realloc(ptr, size); -} - -static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { - (void)cache; - __libc_free(ptr); -} - -InternalAllocator *internal_allocator() { - return 0; -} - -#else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) - static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; static atomic_uint8_t internal_allocator_initialized; static StaticSpinMutex internal_alloc_init_mu; @@ -135,8 +75,6 @@ static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { internal_allocator()->Deallocate(cache, ptr); } -#endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) - static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) { SetAllocatorOutOfMemory(); Report("FATAL: %s: internal allocator is out of memory trying to allocate " @@ -187,6 +125,16 @@ void InternalFree(void *addr, InternalAllocatorCache *cache) { RawInternalFree(addr, cache); } +void InternalAllocatorLock() NO_THREAD_SAFETY_ANALYSIS { + internal_allocator_cache_mu.Lock(); + internal_allocator()->ForceLock(); +} + +void InternalAllocatorUnlock() NO_THREAD_SAFETY_ANALYSIS { + internal_allocator()->ForceUnlock(); + internal_allocator_cache_mu.Unlock(); +} + // LowLevelAllocator constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h index 32849036fd04..38994736877a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -48,6 +48,8 @@ void *InternalReallocArray(void *p, uptr count, uptr size, void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache = nullptr); void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); +void InternalAllocatorLock(); +void InternalAllocatorUnlock(); InternalAllocator *internal_allocator(); } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 065154496eb5..6ec6bb4bd856 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -460,6 +460,10 @@ template <class T> constexpr T Max(T a, T b) { return a > b ? a : b; } +template <class T> +constexpr T Abs(T a) { + return a < 0 ? -a : a; +} template<class T> void Swap(T& a, T& b) { T tmp = a; a = b; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index abb38ccfa15d..d219734fa0a3 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -21,7 +21,7 @@ // COMMON_INTERCEPTOR_FD_RELEASE // COMMON_INTERCEPTOR_FD_ACCESS // COMMON_INTERCEPTOR_SET_THREAD_NAME -// COMMON_INTERCEPTOR_ON_DLOPEN +// COMMON_INTERCEPTOR_DLOPEN // COMMON_INTERCEPTOR_ON_EXIT // COMMON_INTERCEPTOR_MUTEX_PRE_LOCK // COMMON_INTERCEPTOR_MUTEX_POST_LOCK @@ -206,9 +206,9 @@ extern const short *_tolower_tab_; COMMON_INTERCEPTOR_READ_RANGE((ctx), (s), \ common_flags()->strict_string_checks ? (internal_strlen(s)) + 1 : (n) ) -#ifndef COMMON_INTERCEPTOR_ON_DLOPEN -#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \ - CheckNoDeepBind(filename, flag); +#ifndef COMMON_INTERCEPTOR_DLOPEN +#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ + ({ CheckNoDeepBind(filename, flag); REAL(dlopen)(filename, flag); }) #endif #ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE @@ -6380,8 +6380,7 @@ INTERCEPTOR(void*, dlopen, const char *filename, int flag) { void *ctx; COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag); if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0); - COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag); - void *res = REAL(dlopen)(filename, flag); + void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag); Symbolizer::GetOrInit()->InvalidateModuleList(); COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res); return res; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_dense_map.h b/compiler-rt/lib/sanitizer_common/sanitizer_dense_map.h index 3fa6af76ce29..046d77dddc9c 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_dense_map.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_dense_map.h @@ -226,28 +226,26 @@ class DenseMapBase { return FindAndConstruct(__sanitizer::move(Key)).second; } - /// Equality comparison for DenseMap. + /// Iterate over active entries of the container. /// - /// Iterates over elements of LHS confirming that each (key, value) pair in - /// LHS is also in RHS, and that no additional pairs are in RHS. Equivalent to - /// N calls to RHS.find and N value comparisons. Amortized complexity is - /// linear, worst case is O(N^2) (if every hash collides). - bool operator==(const DenseMapBase &RHS) const { - if (size() != RHS.size()) - return false; - + /// Function can return fast to stop the process. + template <class Fn> + void forEach(Fn fn) { const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); for (auto *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { const KeyT K = P->getFirst(); if (!KeyInfoT::isEqual(K, EmptyKey) && !KeyInfoT::isEqual(K, TombstoneKey)) { - const auto *I = RHS.find(K); - if (!I || P->getSecond() != I->getSecond()) - return false; + if (!fn(*P)) + return; } } + } - return true; + template <class Fn> + void forEach(Fn fn) const { + const_cast<DenseMapBase *>(this)->forEach( + [&](const value_type &KV) { return fn(KV); }); } protected: @@ -524,6 +522,35 @@ class DenseMapBase { } }; +/// Equality comparison for DenseMap. +/// +/// Iterates over elements of LHS confirming that each (key, value) pair in LHS +/// is also in RHS, and that no additional pairs are in RHS. +/// Equivalent to N calls to RHS.find and N value comparisons. Amortized +/// complexity is linear, worst case is O(N^2) (if every hash collides). +template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT, + typename BucketT> +bool operator==( + const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS, + const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) { + if (LHS.size() != RHS.size()) + return false; + + bool R = true; + LHS.forEach( + [&](const typename DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, + BucketT>::value_type &KV) -> bool { + const auto *I = RHS.find(KV.first); + if (!I || I->second != KV.second) { + R = false; + return false; + } + return true; + }); + + return R; +} + /// Inequality comparison for DenseMap. /// /// Equivalent to !(LHS == RHS). See operator== for performance notes. diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_dense_map_info.h b/compiler-rt/lib/sanitizer_common/sanitizer_dense_map_info.h index 85c6427906c1..f4640369ae58 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_dense_map_info.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_dense_map_info.h @@ -18,7 +18,7 @@ namespace __sanitizer { namespace detail { /// Simplistic combination of 32-bit hash values into 32-bit hash values. -static inline unsigned combineHashValue(unsigned a, unsigned b) { +static constexpr unsigned combineHashValue(unsigned a, unsigned b) { u64 key = (u64)a << 32 | (u64)b; key += ~(key << 32); key ^= (key >> 22); @@ -37,18 +37,19 @@ template <typename KeyT, typename ValueT> struct DenseMapPair { KeyT first = {}; ValueT second = {}; - DenseMapPair() = default; - DenseMapPair(const KeyT &f, const ValueT &s) : first(f), second(s) {} + constexpr DenseMapPair() = default; + constexpr DenseMapPair(const KeyT &f, const ValueT &s) + : first(f), second(s) {} template <typename KeyT2, typename ValueT2> - DenseMapPair(KeyT2 &&f, ValueT2 &&s) + constexpr DenseMapPair(KeyT2 &&f, ValueT2 &&s) : first(__sanitizer::forward<KeyT2>(f)), second(__sanitizer::forward<ValueT2>(s)) {} - DenseMapPair(const DenseMapPair &other) = default; - DenseMapPair &operator=(const DenseMapPair &other) = default; - DenseMapPair(DenseMapPair &&other) = default; - DenseMapPair &operator=(DenseMapPair &&other) = default; + constexpr DenseMapPair(const DenseMapPair &other) = default; + constexpr DenseMapPair &operator=(const DenseMapPair &other) = default; + constexpr DenseMapPair(DenseMapPair &&other) = default; + constexpr DenseMapPair &operator=(DenseMapPair &&other) = default; KeyT &getFirst() { return first; } const KeyT &getFirst() const { return first; } @@ -60,8 +61,8 @@ struct DenseMapPair { template <typename T> struct DenseMapInfo { - // static inline T getEmptyKey(); - // static inline T getTombstoneKey(); + // static T getEmptyKey(); + // static T getTombstoneKey(); // static unsigned getHashValue(const T &Val); // static bool isEqual(const T &LHS, const T &RHS); }; @@ -79,43 +80,50 @@ struct DenseMapInfo<T *> { // "Log2MaxAlign bits of alignment"); static constexpr uptr Log2MaxAlign = 12; - static inline T *getEmptyKey() { + static constexpr T *getEmptyKey() { uptr Val = static_cast<uptr>(-1); Val <<= Log2MaxAlign; return reinterpret_cast<T *>(Val); } - static inline T *getTombstoneKey() { + static constexpr T *getTombstoneKey() { uptr Val = static_cast<uptr>(-2); Val <<= Log2MaxAlign; return reinterpret_cast<T *>(Val); } - static unsigned getHashValue(const T *PtrVal) { + static constexpr unsigned getHashValue(const T *PtrVal) { return (unsigned((uptr)PtrVal) >> 4) ^ (unsigned((uptr)PtrVal) >> 9); } - static bool isEqual(const T *LHS, const T *RHS) { return LHS == RHS; } + static constexpr bool isEqual(const T *LHS, const T *RHS) { + return LHS == RHS; + } }; // Provide DenseMapInfo for chars. template <> struct DenseMapInfo<char> { - static inline char getEmptyKey() { return ~0; } - static inline char getTombstoneKey() { return ~0 - 1; } - static unsigned getHashValue(const char &Val) { return Val * 37U; } + static constexpr char getEmptyKey() { return ~0; } + static constexpr char getTombstoneKey() { return ~0 - 1; } + static constexpr unsigned getHashValue(const char &Val) { return Val * 37U; } - static bool isEqual(const char &LHS, const char &RHS) { return LHS == RHS; } + static constexpr bool isEqual(const char &LHS, const char &RHS) { + return LHS == RHS; + } }; // Provide DenseMapInfo for unsigned chars. template <> struct DenseMapInfo<unsigned char> { - static inline unsigned char getEmptyKey() { return ~0; } - static inline unsigned char getTombstoneKey() { return ~0 - 1; } - static unsigned getHashValue(const unsigned char &Val) { return Val * 37U; } + static constexpr unsigned char getEmptyKey() { return ~0; } + static constexpr unsigned char getTombstoneKey() { return ~0 - 1; } + static constexpr unsigned getHashValue(const unsigned char &Val) { + return Val * 37U; + } - static bool isEqual(const unsigned char &LHS, const unsigned char &RHS) { + static constexpr bool isEqual(const unsigned char &LHS, + const unsigned char &RHS) { return LHS == RHS; } }; @@ -123,11 +131,14 @@ struct DenseMapInfo<unsigned char> { // Provide DenseMapInfo for unsigned shorts. template <> struct DenseMapInfo<unsigned short> { - static inline unsigned short getEmptyKey() { return 0xFFFF; } - static inline unsigned short getTombstoneKey() { return 0xFFFF - 1; } - static unsigned getHashValue(const unsigned short &Val) { return Val * 37U; } + static constexpr unsigned short getEmptyKey() { return 0xFFFF; } + static constexpr unsigned short getTombstoneKey() { return 0xFFFF - 1; } + static constexpr unsigned getHashValue(const unsigned short &Val) { + return Val * 37U; + } - static bool isEqual(const unsigned short &LHS, const unsigned short &RHS) { + static constexpr bool isEqual(const unsigned short &LHS, + const unsigned short &RHS) { return LHS == RHS; } }; @@ -135,11 +146,13 @@ struct DenseMapInfo<unsigned short> { // Provide DenseMapInfo for unsigned ints. template <> struct DenseMapInfo<unsigned> { - static inline unsigned getEmptyKey() { return ~0U; } - static inline unsigned getTombstoneKey() { return ~0U - 1; } - static unsigned getHashValue(const unsigned &Val) { return Val * 37U; } + static constexpr unsigned getEmptyKey() { return ~0U; } + static constexpr unsigned getTombstoneKey() { return ~0U - 1; } + static constexpr unsigned getHashValue(const unsigned &Val) { + return Val * 37U; + } - static bool isEqual(const unsigned &LHS, const unsigned &RHS) { + static constexpr bool isEqual(const unsigned &LHS, const unsigned &RHS) { return LHS == RHS; } }; @@ -147,14 +160,15 @@ struct DenseMapInfo<unsigned> { // Provide DenseMapInfo for unsigned longs. template <> struct DenseMapInfo<unsigned long> { - static inline unsigned long getEmptyKey() { return ~0UL; } - static inline unsigned long getTombstoneKey() { return ~0UL - 1L; } + static constexpr unsigned long getEmptyKey() { return ~0UL; } + static constexpr unsigned long getTombstoneKey() { return ~0UL - 1L; } - static unsigned getHashValue(const unsigned long &Val) { + static constexpr unsigned getHashValue(const unsigned long &Val) { return (unsigned)(Val * 37UL); } - static bool isEqual(const unsigned long &LHS, const unsigned long &RHS) { + static constexpr bool isEqual(const unsigned long &LHS, + const unsigned long &RHS) { return LHS == RHS; } }; @@ -162,15 +176,15 @@ struct DenseMapInfo<unsigned long> { // Provide DenseMapInfo for unsigned long longs. template <> struct DenseMapInfo<unsigned long long> { - static inline unsigned long long getEmptyKey() { return ~0ULL; } - static inline unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; } + static constexpr unsigned long long getEmptyKey() { return ~0ULL; } + static constexpr unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; } - static unsigned getHashValue(const unsigned long long &Val) { + static constexpr unsigned getHashValue(const unsigned long long &Val) { return (unsigned)(Val * 37ULL); } - static bool isEqual(const unsigned long long &LHS, - const unsigned long long &RHS) { + static constexpr bool isEqual(const unsigned long long &LHS, + const unsigned long long &RHS) { return LHS == RHS; } }; @@ -178,51 +192,59 @@ struct DenseMapInfo<unsigned long long> { // Provide DenseMapInfo for shorts. template <> struct DenseMapInfo<short> { - static inline short getEmptyKey() { return 0x7FFF; } - static inline short getTombstoneKey() { return -0x7FFF - 1; } - static unsigned getHashValue(const short &Val) { return Val * 37U; } - static bool isEqual(const short &LHS, const short &RHS) { return LHS == RHS; } + static constexpr short getEmptyKey() { return 0x7FFF; } + static constexpr short getTombstoneKey() { return -0x7FFF - 1; } + static constexpr unsigned getHashValue(const short &Val) { return Val * 37U; } + static constexpr bool isEqual(const short &LHS, const short &RHS) { + return LHS == RHS; + } }; // Provide DenseMapInfo for ints. template <> struct DenseMapInfo<int> { - static inline int getEmptyKey() { return 0x7fffffff; } - static inline int getTombstoneKey() { return -0x7fffffff - 1; } - static unsigned getHashValue(const int &Val) { return (unsigned)(Val * 37U); } + static constexpr int getEmptyKey() { return 0x7fffffff; } + static constexpr int getTombstoneKey() { return -0x7fffffff - 1; } + static constexpr unsigned getHashValue(const int &Val) { + return (unsigned)(Val * 37U); + } - static bool isEqual(const int &LHS, const int &RHS) { return LHS == RHS; } + static constexpr bool isEqual(const int &LHS, const int &RHS) { + return LHS == RHS; + } }; // Provide DenseMapInfo for longs. template <> struct DenseMapInfo<long> { - static inline long getEmptyKey() { + static constexpr long getEmptyKey() { return (1UL << (sizeof(long) * 8 - 1)) - 1UL; } - static inline long getTombstoneKey() { return getEmptyKey() - 1L; } + static constexpr long getTombstoneKey() { return getEmptyKey() - 1L; } - static unsigned getHashValue(const long &Val) { + static constexpr unsigned getHashValue(const long &Val) { return (unsigned)(Val * 37UL); } - static bool isEqual(const long &LHS, const long &RHS) { return LHS == RHS; } + static constexpr bool isEqual(const long &LHS, const long &RHS) { + return LHS == RHS; + } }; // Provide DenseMapInfo for long longs. template <> struct DenseMapInfo<long long> { - static inline long long getEmptyKey() { return 0x7fffffffffffffffLL; } - static inline long long getTombstoneKey() { + static constexpr long long getEmptyKey() { return 0x7fffffffffffffffLL; } + static constexpr long long getTombstoneKey() { return -0x7fffffffffffffffLL - 1; } - static unsigned getHashValue(const long long &Val) { + static constexpr unsigned getHashValue(const long long &Val) { return (unsigned)(Val * 37ULL); } - static bool isEqual(const long long &LHS, const long long &RHS) { + static constexpr bool isEqual(const long long &LHS, const long long &RHS) { return LHS == RHS; } }; @@ -234,22 +256,22 @@ struct DenseMapInfo<detail::DenseMapPair<T, U>> { using FirstInfo = DenseMapInfo<T>; using SecondInfo = DenseMapInfo<U>; - static inline Pair getEmptyKey() { + static constexpr Pair getEmptyKey() { return detail::DenseMapPair<T, U>(FirstInfo::getEmptyKey(), SecondInfo::getEmptyKey()); } - static inline Pair getTombstoneKey() { + static constexpr Pair getTombstoneKey() { return detail::DenseMapPair<T, U>(FirstInfo::getTombstoneKey(), SecondInfo::getTombstoneKey()); } - static unsigned getHashValue(const Pair &PairVal) { + static constexpr unsigned getHashValue(const Pair &PairVal) { return detail::combineHashValue(FirstInfo::getHashValue(PairVal.first), SecondInfo::getHashValue(PairVal.second)); } - static bool isEqual(const Pair &LHS, const Pair &RHS) { + static constexpr bool isEqual(const Pair &LHS, const Pair &RHS) { return FirstInfo::isEqual(LHS.first, RHS.first) && SecondInfo::isEqual(LHS.second, RHS.second); } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp index c7b30d988365..9b5f6f1da1a1 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp @@ -274,6 +274,15 @@ void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { UNIMPLEMENTED(); } +bool MprotectNoAccess(uptr addr, uptr size) { + return _zx_vmar_protect(_zx_vmar_root_self(), 0, Addr, Size) == ZX_OK; +} + +bool MprotectReadOnly(uptr addr, uptr size) { + return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, Addr, Size) == + ZX_OK; +} + void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, const char *mem_type) { CHECK_GE(size, GetPageSize()); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_leb128.h b/compiler-rt/lib/sanitizer_common/sanitizer_leb128.h new file mode 100644 index 000000000000..553550d29552 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_leb128.h @@ -0,0 +1,87 @@ +//===-- sanitizer_leb128.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_LEB128_H +#define SANITIZER_LEB128_H + +#include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" + +namespace __sanitizer { + +template <typename T, typename It> +It EncodeSLEB128(T value, It begin, It end) { + bool more; + do { + u8 byte = value & 0x7f; + // NOTE: this assumes that this signed shift is an arithmetic right shift. + value >>= 7; + more = !((((value == 0) && ((byte & 0x40) == 0)) || + ((value == -1) && ((byte & 0x40) != 0)))); + if (more) + byte |= 0x80; + if (UNLIKELY(begin == end)) + break; + *(begin++) = byte; + } while (more); + return begin; +} + +template <typename T, typename It> +It DecodeSLEB128(It begin, It end, T* v) { + T value = 0; + unsigned shift = 0; + u8 byte; + do { + if (UNLIKELY(begin == end)) + return begin; + byte = *(begin++); + T slice = byte & 0x7f; + value |= slice << shift; + shift += 7; + } while (byte >= 128); + if (shift < 64 && (byte & 0x40)) + value |= (-1ULL) << shift; + *v = value; + return begin; +} + +template <typename T, typename It> +It EncodeULEB128(T value, It begin, It end) { + do { + u8 byte = value & 0x7f; + value >>= 7; + if (value) + byte |= 0x80; + if (UNLIKELY(begin == end)) + break; + *(begin++) = byte; + } while (value); + return begin; +} + +template <typename T, typename It> +It DecodeULEB128(It begin, It end, T* v) { + T value = 0; + unsigned shift = 0; + u8 byte; + do { + if (UNLIKELY(begin == end)) + return begin; + byte = *(begin++); + T slice = byte & 0x7f; + value += slice << shift; + shift += 7; + } while (byte >= 128); + *v = value; + return begin; +} + +} // namespace __sanitizer + +#endif // SANITIZER_LEB128_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp index 596037d77222..2d787332a445 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp @@ -1760,6 +1760,8 @@ HandleSignalMode GetHandleSignalMode(int signum) { #if !SANITIZER_GO void *internal_start_thread(void *(*func)(void *arg), void *arg) { + if (&real_pthread_create == 0) + return nullptr; // Start the thread with signals blocked, otherwise it can steal user signals. ScopedBlockSignals block(nullptr); void *th; @@ -1768,7 +1770,8 @@ void *internal_start_thread(void *(*func)(void *arg), void *arg) { } void internal_join_thread(void *th) { - real_pthread_join(th, nullptr); + if (&real_pthread_join) + real_pthread_join(th, nullptr); } #else void *internal_start_thread(void *(*func)(void *), void *arg) { return 0; } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp index b67203d4c10e..f9b5c531aeee 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp @@ -265,30 +265,32 @@ int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, static fd_t internal_spawn_impl(const char *argv[], const char *envp[], pid_t *pid) { - fd_t master_fd = kInvalidFd; - fd_t slave_fd = kInvalidFd; + fd_t primary_fd = kInvalidFd; + fd_t secondary_fd = kInvalidFd; auto fd_closer = at_scope_exit([&] { - internal_close(master_fd); - internal_close(slave_fd); + internal_close(primary_fd); + internal_close(secondary_fd); }); // We need a new pseudoterminal to avoid buffering problems. The 'atos' tool // in particular detects when it's talking to a pipe and forgets to flush the // output stream after sending a response. - master_fd = posix_openpt(O_RDWR); - if (master_fd == kInvalidFd) return kInvalidFd; + primary_fd = posix_openpt(O_RDWR); + if (primary_fd == kInvalidFd) + return kInvalidFd; - int res = grantpt(master_fd) || unlockpt(master_fd); + int res = grantpt(primary_fd) || unlockpt(primary_fd); if (res != 0) return kInvalidFd; // Use TIOCPTYGNAME instead of ptsname() to avoid threading problems. - char slave_pty_name[128]; - res = ioctl(master_fd, TIOCPTYGNAME, slave_pty_name); + char secondary_pty_name[128]; + res = ioctl(primary_fd, TIOCPTYGNAME, secondary_pty_name); if (res == -1) return kInvalidFd; - slave_fd = internal_open(slave_pty_name, O_RDWR); - if (slave_fd == kInvalidFd) return kInvalidFd; + secondary_fd = internal_open(secondary_pty_name, O_RDWR); + if (secondary_fd == kInvalidFd) + return kInvalidFd; // File descriptor actions posix_spawn_file_actions_t acts; @@ -299,9 +301,9 @@ static fd_t internal_spawn_impl(const char *argv[], const char *envp[], posix_spawn_file_actions_destroy(&acts); }); - res = posix_spawn_file_actions_adddup2(&acts, slave_fd, STDIN_FILENO) || - posix_spawn_file_actions_adddup2(&acts, slave_fd, STDOUT_FILENO) || - posix_spawn_file_actions_addclose(&acts, slave_fd); + res = posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDIN_FILENO) || + posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDOUT_FILENO) || + posix_spawn_file_actions_addclose(&acts, secondary_fd); if (res != 0) return kInvalidFd; // Spawn attributes @@ -326,14 +328,14 @@ static fd_t internal_spawn_impl(const char *argv[], const char *envp[], // Disable echo in the new terminal, disable CR. struct termios termflags; - tcgetattr(master_fd, &termflags); + tcgetattr(primary_fd, &termflags); termflags.c_oflag &= ~ONLCR; termflags.c_lflag &= ~ECHO; - tcsetattr(master_fd, TCSANOW, &termflags); + tcsetattr(primary_fd, TCSANOW, &termflags); - // On success, do not close master_fd on scope exit. - fd_t fd = master_fd; - master_fd = kInvalidFd; + // On success, do not close primary_fd on scope exit. + fd_t fd = primary_fd; + primary_fd = kInvalidFd; return fd; } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp index ad88e2bbbefc..b1c15d8c2834 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp @@ -14,78 +14,187 @@ namespace __sanitizer { -static constexpr u32 kStackSizeBits = 16; +namespace { +struct StackTraceHeader { + static constexpr u32 kStackSizeBits = 8; -StackStore::Id StackStore::Store(const StackTrace &trace) { - uptr *stack_trace = Alloc(trace.size + 1); - CHECK_LT(trace.size, 1 << kStackSizeBits); - *stack_trace = trace.size + (trace.tag << kStackSizeBits); - internal_memcpy(stack_trace + 1, trace.trace, trace.size * sizeof(uptr)); - return reinterpret_cast<StackStore::Id>(stack_trace); + u8 size; + u8 tag; + explicit StackTraceHeader(const StackTrace &trace) + : size(Min<uptr>(trace.size, (1u << 8) - 1)), tag(trace.tag) { + CHECK_EQ(trace.tag, static_cast<uptr>(tag)); + } + explicit StackTraceHeader(uptr h) + : size(h & ((1 << kStackSizeBits) - 1)), tag(h >> kStackSizeBits) {} + + uptr ToUptr() const { + return static_cast<uptr>(size) | (static_cast<uptr>(tag) << kStackSizeBits); + } +}; +} // namespace + +StackStore::Id StackStore::Store(const StackTrace &trace, uptr *pack) { + if (!trace.size && !trace.tag) + return 0; + StackTraceHeader h(trace); + uptr idx = 0; + *pack = 0; + uptr *stack_trace = Alloc(h.size + 1, &idx, pack); + *stack_trace = h.ToUptr(); + internal_memcpy(stack_trace + 1, trace.trace, h.size * sizeof(uptr)); + *pack += blocks_[GetBlockIdx(idx)].Stored(h.size + 1); + return OffsetToId(idx); } StackTrace StackStore::Load(Id id) { - const uptr *stack_trace = reinterpret_cast<const uptr *>(id); - uptr size = *stack_trace & ((1 << kStackSizeBits) - 1); - uptr tag = *stack_trace >> kStackSizeBits; - return StackTrace(stack_trace + 1, size, tag); + if (!id) + return {}; + uptr idx = IdToOffset(id); + uptr block_idx = GetBlockIdx(idx); + CHECK_LT(block_idx, ARRAY_SIZE(blocks_)); + const uptr *stack_trace = blocks_[block_idx].GetOrUnpack(); + if (!stack_trace) + return {}; + stack_trace += GetInBlockIdx(idx); + StackTraceHeader h(*stack_trace); + return StackTrace(stack_trace + 1, h.size, h.tag); } -uptr *StackStore::TryAlloc(uptr count) { - // Optimisic lock-free allocation, essentially try to bump the region ptr. +uptr StackStore::Allocated() const { + uptr next_block = GetBlockIdx( + RoundUpTo(atomic_load_relaxed(&total_frames_), kBlockSizeFrames)); + uptr res = 0; + for (uptr i = 0; i < next_block; ++i) res += blocks_[i].Allocated(); + return res + sizeof(*this); +} + +uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) { for (;;) { - uptr cmp = atomic_load(®ion_pos_, memory_order_acquire); - uptr end = atomic_load(®ion_end_, memory_order_acquire); - uptr size = count * sizeof(uptr); - if (cmp == 0 || cmp + size > end) - return nullptr; - if (atomic_compare_exchange_weak(®ion_pos_, &cmp, cmp + size, - memory_order_acquire)) - return reinterpret_cast<uptr *>(cmp); + // Optimisic lock-free allocation, essentially try to bump the + // total_frames_. + uptr start = atomic_fetch_add(&total_frames_, count, memory_order_relaxed); + uptr block_idx = GetBlockIdx(start); + uptr last_idx = GetBlockIdx(start + count - 1); + if (LIKELY(block_idx == last_idx)) { + // Fits into the a single block. + CHECK_LT(block_idx, ARRAY_SIZE(blocks_)); + *idx = start; + return blocks_[block_idx].GetOrCreate() + GetInBlockIdx(start); + } + + // Retry. We can't use range allocated in two different blocks. + CHECK_LE(count, kBlockSizeFrames); + uptr in_first = kBlockSizeFrames - GetInBlockIdx(start); + // Mark tail/head of these blocks as "stored".to avoid waiting before we can + // Pack(). + *pack += blocks_[block_idx].Stored(in_first); + *pack += blocks_[last_idx].Stored(count - in_first); } } -uptr *StackStore::Alloc(uptr count) { - // First, try to allocate optimisitically. - uptr *s = TryAlloc(count); - if (LIKELY(s)) - return s; - return RefillAndAlloc(count); +uptr StackStore::Pack(Compression type) { + uptr res = 0; + for (BlockInfo &b : blocks_) res += b.Pack(type); + return res; +} + +void StackStore::TestOnlyUnmap() { + for (BlockInfo &b : blocks_) b.TestOnlyUnmap(); + internal_memset(this, 0, sizeof(*this)); } -uptr *StackStore::RefillAndAlloc(uptr count) { - // If failed, lock, retry and alloc new superblock. +uptr *StackStore::BlockInfo::Get() const { + // Idiomatic double-checked locking uses memory_order_acquire here. But + // relaxed is find for us, justification is similar to + // TwoLevelMap::GetOrCreate. + return reinterpret_cast<uptr *>(atomic_load_relaxed(&data_)); +} + +uptr *StackStore::BlockInfo::Create() { SpinMutexLock l(&mtx_); - for (;;) { - uptr *s = TryAlloc(count); - if (s) - return s; - atomic_store(®ion_pos_, 0, memory_order_relaxed); - uptr size = count * sizeof(uptr) + sizeof(BlockInfo); - uptr allocsz = RoundUpTo(Max<uptr>(size, 64u * 1024u), GetPageSizeCached()); - uptr mem = (uptr)MmapOrDie(allocsz, "stack depot"); - BlockInfo *new_block = (BlockInfo *)(mem + allocsz) - 1; - new_block->next = curr_; - new_block->ptr = mem; - new_block->size = allocsz; - curr_ = new_block; + uptr *ptr = Get(); + if (!ptr) { + ptr = reinterpret_cast<uptr *>( + MmapNoReserveOrDie(kBlockSizeBytes, "StackStore")); + atomic_store(&data_, reinterpret_cast<uptr>(ptr), memory_order_release); + } + return ptr; +} + +uptr *StackStore::BlockInfo::GetOrCreate() { + uptr *ptr = Get(); + if (LIKELY(ptr)) + return ptr; + return Create(); +} - atomic_fetch_add(&mapped_size_, allocsz, memory_order_relaxed); +uptr *StackStore::BlockInfo::GetOrUnpack() { + SpinMutexLock l(&mtx_); + switch (state) { + case State::Storing: + state = State::Unpacked; + FALLTHROUGH; + case State::Unpacked: + return Get(); + case State::Packed: + break; + } + + uptr *ptr = Get(); + CHECK_NE(nullptr, ptr); + // Fake unpacking. + for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i]; + state = State::Unpacked; + return Get(); +} + +uptr StackStore::BlockInfo::Pack(Compression type) { + if (type == Compression::None) + return 0; - allocsz -= sizeof(BlockInfo); - atomic_store(®ion_end_, mem + allocsz, memory_order_release); - atomic_store(®ion_pos_, mem, memory_order_release); + SpinMutexLock l(&mtx_); + switch (state) { + case State::Unpacked: + case State::Packed: + return 0; + case State::Storing: + break; } + + uptr *ptr = Get(); + if (!ptr || !Stored(0)) + return 0; + + // Fake packing. + for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i]; + state = State::Packed; + return kBlockSizeBytes - kBlockSizeBytes / 10; } -void StackStore::TestOnlyUnmap() { - while (curr_) { - uptr mem = curr_->ptr; - uptr allocsz = curr_->size; - curr_ = curr_->next; - UnmapOrDie((void *)mem, allocsz); +uptr StackStore::BlockInfo::Allocated() const { + SpinMutexLock l(&mtx_); + switch (state) { + case State::Packed: + return kBlockSizeBytes / 10; + case State::Unpacked: + case State::Storing: + return kBlockSizeBytes; } - internal_memset(this, 0, sizeof(*this)); +} + +void StackStore::BlockInfo::TestOnlyUnmap() { + if (uptr *ptr = Get()) + UnmapOrDie(ptr, StackStore::kBlockSizeBytes); +} + +bool StackStore::BlockInfo::Stored(uptr n) { + return n + atomic_fetch_add(&stored_, n, memory_order_release) == + kBlockSizeFrames; +} + +bool StackStore::BlockInfo::IsPacked() const { + SpinMutexLock l(&mtx_); + return state == State::Packed; } } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h index b5bbdccc20b1..e0bc4e9c4a45 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h @@ -10,6 +10,7 @@ #define SANITIZER_STACK_STORE_H #include "sanitizer_atomic.h" +#include "sanitizer_common.h" #include "sanitizer_internal_defs.h" #include "sanitizer_mutex.h" #include "sanitizer_stacktrace.h" @@ -17,32 +18,91 @@ namespace __sanitizer { class StackStore { + static constexpr uptr kBlockSizeFrames = 0x100000; + static constexpr uptr kBlockCount = 0x1000; + static constexpr uptr kBlockSizeBytes = kBlockSizeFrames * sizeof(uptr); + public: + enum class Compression : u8 { + None = 0, + Test, + }; + constexpr StackStore() = default; - using Id = uptr; + using Id = u32; // Enough for 2^32 * sizeof(uptr) bytes of traces. + static_assert(u64(kBlockCount) * kBlockSizeFrames == 1ull << (sizeof(Id) * 8), + ""); - Id Store(const StackTrace &trace); + Id Store(const StackTrace &trace, + uptr *pack /* number of blocks completed by this call */); StackTrace Load(Id id); - uptr Allocated() const { return atomic_load_relaxed(&mapped_size_); } + uptr Allocated() const; + + // Packs all blocks which don't expect any more writes. A block is going to be + // packed once. As soon trace from that block was requested, it will unpack + // and stay unpacked after that. + // Returns the number of released bytes. + uptr Pack(Compression type); void TestOnlyUnmap(); private: - uptr *Alloc(uptr count = 1); - uptr *TryAlloc(uptr count); - uptr *RefillAndAlloc(uptr count); - mutable StaticSpinMutex mtx_ = {}; // Protects alloc of new blocks. - atomic_uintptr_t region_pos_ = {}; // Region allocator for Node's. - atomic_uintptr_t region_end_ = {}; - atomic_uintptr_t mapped_size_ = {}; + friend class StackStoreTest; + static constexpr uptr GetBlockIdx(uptr frame_idx) { + return frame_idx / kBlockSizeFrames; + } + + static constexpr uptr GetInBlockIdx(uptr frame_idx) { + return frame_idx % kBlockSizeFrames; + } + + static constexpr uptr IdToOffset(Id id) { + CHECK_NE(id, 0); + return id - 1; // Avoid zero as id. + } + + static constexpr uptr OffsetToId(Id id) { + // This makes UINT32_MAX to 0 and it will be retrived as and empty stack. + // But this is not a problem as we will not be able to store anything after + // that anyway. + return id + 1; // Avoid zero as id. + } - struct BlockInfo { - const BlockInfo *next; - uptr ptr; - uptr size; + uptr *Alloc(uptr count, uptr *idx, uptr *pack); + + // Total number of allocated frames. + atomic_uintptr_t total_frames_ = {}; + + // Each block will hold pointer to exactly kBlockSizeFrames. + class BlockInfo { + atomic_uintptr_t data_; + // Counter to track store progress to know when we can Pack() the block. + atomic_uint32_t stored_; + // Protects alloc of new blocks. + mutable StaticSpinMutex mtx_; + + enum class State : u8 { + Storing = 0, + Packed, + Unpacked, + }; + State state GUARDED_BY(mtx_); + + uptr *Create(); + + public: + uptr *Get() const; + uptr *GetOrCreate(); + uptr *GetOrUnpack(); + uptr Pack(Compression type); + uptr Allocated() const; + void TestOnlyUnmap(); + bool Stored(uptr n); + bool IsPacked() const; }; - const BlockInfo *curr_ = nullptr; + + BlockInfo blocks_[kBlockCount] = {}; }; } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp index e203b2cc4c89..527221b0c85c 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp @@ -23,6 +23,7 @@ struct StackDepotNode { using hash_type = u64; hash_type stack_hash; u32 link; + StackStore::Id store_id; static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20; @@ -53,11 +54,6 @@ static StackStore stackStore; typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog> StackDepot; static StackDepot theDepot; -// Keep rarely accessed stack traces out of frequently access nodes to improve -// caching efficiency. -static TwoLevelMap<StackStore::Id, StackDepot::kNodesSize1, - StackDepot::kNodesSize2> - storeIds; // Keep mutable data out of frequently access nodes to improve caching // efficiency. static TwoLevelMap<atomic_uint32_t, StackDepot::kNodesSize1, @@ -73,17 +69,18 @@ void StackDepotHandle::inc_use_count_unsafe() { } uptr StackDepotNode::allocated() { - return stackStore.Allocated() + storeIds.MemoryUsage() + - useCounts.MemoryUsage(); + return stackStore.Allocated() + useCounts.MemoryUsage(); } void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) { stack_hash = hash; - storeIds[id] = stackStore.Store(args); + uptr pack = 0; + store_id = stackStore.Store(args, &pack); + if (pack) + stackStore.Pack(StackStore::Compression::None); } StackDepotNode::args_type StackDepotNode::load(u32 id) const { - StackStore::Id store_id = storeIds[id]; if (!store_id) return {}; return stackStore.Load(store_id); @@ -121,7 +118,6 @@ StackDepotHandle StackDepotNode::get_handle(u32 id) { void StackDepotTestOnlyUnmap() { theDepot.TestOnlyUnmap(); - storeIds.TestOnlyUnmap(); stackStore.TestOnlyUnmap(); } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h index 11c6154b09ea..aebd504669d2 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h @@ -20,7 +20,7 @@ namespace __sanitizer { struct BufferedStackTrace; -static const u32 kStackTraceMax = 256; +static const u32 kStackTraceMax = 255; #if SANITIZER_LINUX && defined(__mips__) # define SANITIZER_CAN_FAST_UNWIND 0 diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp index c3607dbed23e..1a31ce02af4c 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp @@ -337,6 +337,11 @@ bool MprotectNoAccess(uptr addr, uptr size) { return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection); } +bool MprotectReadOnly(uptr addr, uptr size) { + DWORD old_protection; + return VirtualProtect((LPVOID)addr, size, PAGE_READONLY, &old_protection); +} + void ReleaseMemoryPagesToOS(uptr beg, uptr end) { uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()), end_aligned = RoundDownTo(end, GetPageSizeCached()); diff --git a/compiler-rt/lib/tsan/rtl/tsan_defs.h b/compiler-rt/lib/tsan/rtl/tsan_defs.h index fe0c1da31599..4712c2be1813 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_defs.h +++ b/compiler-rt/lib/tsan/rtl/tsan_defs.h @@ -228,6 +228,7 @@ enum MutexType { MutexTypeFired, MutexTypeRacy, MutexTypeGlobalProc, + MutexTypeInternalAlloc, }; } // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 25dbe487b280..cf3dc90d96a1 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -177,6 +177,7 @@ struct ThreadSignalContext { struct AtExitCtx { void (*f)(); void *arg; + uptr pc; }; // InterceptorContext holds all global data required for interceptors. @@ -367,7 +368,10 @@ TSAN_INTERCEPTOR(int, pause, int fake) { return BLOCK_REAL(pause)(fake); } -static void at_exit_wrapper() { +// Note: we specifically call the function in such strange way +// with "installed_at" because in reports it will appear between +// callback frames and the frame that installed the callback. +static void at_exit_callback_installed_at() { AtExitCtx *ctx; { // Ensure thread-safety. @@ -379,15 +383,21 @@ static void at_exit_wrapper() { interceptor_ctx()->AtExitStack.PopBack(); } - Acquire(cur_thread(), (uptr)0, (uptr)ctx); + ThreadState *thr = cur_thread(); + Acquire(thr, ctx->pc, (uptr)ctx); + FuncEntry(thr, ctx->pc); ((void(*)())ctx->f)(); + FuncExit(thr); Free(ctx); } -static void cxa_at_exit_wrapper(void *arg) { - Acquire(cur_thread(), 0, (uptr)arg); +static void cxa_at_exit_callback_installed_at(void *arg) { + ThreadState *thr = cur_thread(); AtExitCtx *ctx = (AtExitCtx*)arg; + Acquire(thr, ctx->pc, (uptr)arg); + FuncEntry(thr, ctx->pc); ((void(*)(void *arg))ctx->f)(ctx->arg); + FuncExit(thr); Free(ctx); } @@ -401,7 +411,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { // We want to setup the atexit callback even if we are in ignored lib // or after fork. SCOPED_INTERCEPTOR_RAW(atexit, f); - return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); + return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0); } #endif @@ -409,7 +419,7 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); - return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); + return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), @@ -417,6 +427,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), auto *ctx = New<AtExitCtx>(); ctx->f = f; ctx->arg = arg; + ctx->pc = pc; Release(thr, pc, (uptr)ctx); // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. @@ -432,25 +443,27 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), // due to atexit_mu held on exit from the calloc interceptor. ScopedIgnoreInterceptors ignore; - res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0); + res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_callback_installed_at, + 0, 0); // Push AtExitCtx on the top of the stack of callback functions if (!res) { interceptor_ctx()->AtExitStack.PushBack(ctx); } } else { - res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso); + res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso); } ThreadIgnoreEnd(thr); return res; } #if !SANITIZER_MAC && !SANITIZER_NETBSD -static void on_exit_wrapper(int status, void *arg) { +static void on_exit_callback_installed_at(int status, void *arg) { ThreadState *thr = cur_thread(); - uptr pc = 0; - Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; + Acquire(thr, ctx->pc, (uptr)arg); + FuncEntry(thr, ctx->pc); ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); + FuncExit(thr); Free(ctx); } @@ -461,11 +474,12 @@ TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { auto *ctx = New<AtExitCtx>(); ctx->f = (void(*)())f; ctx->arg = arg; + ctx->pc = GET_CALLER_PC(); Release(thr, pc, (uptr)ctx); // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. ThreadIgnoreBegin(thr, pc); - int res = REAL(on_exit)(on_exit_wrapper, ctx); + int res = REAL(on_exit)(on_exit_callback_installed_at, ctx); ThreadIgnoreEnd(thr); return res; } @@ -2363,6 +2377,15 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, if (fd >= 0) FdClose(thr, pc, fd); \ } +#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ + ({ \ + CheckNoDeepBind(filename, flag); \ + ThreadIgnoreBegin(thr, 0); \ + void *res = REAL(dlopen)(filename, flag); \ + ThreadIgnoreEnd(thr); \ + res; \ + }) + #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ libignore()->OnLibraryLoaded(filename) diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp index ef97ad0bc94e..a31bebcb6ba9 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp @@ -69,8 +69,17 @@ Allocator *allocator() { struct GlobalProc { Mutex mtx; Processor *proc; + // This mutex represents the internal allocator combined for + // the purposes of deadlock detection. The internal allocator + // uses multiple mutexes, moreover they are locked only occasionally + // and they are spin mutexes which don't support deadlock detection. + // So we use this fake mutex to serve as a substitute for these mutexes. + CheckedMutex internal_alloc_mtx; - GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {} + GlobalProc() + : mtx(MutexTypeGlobalProc), + proc(ProcCreate()), + internal_alloc_mtx(MutexTypeInternalAlloc) {} }; static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64); @@ -78,6 +87,11 @@ GlobalProc *global_proc() { return reinterpret_cast<GlobalProc*>(&global_proc_placeholder); } +static void InternalAllocAccess() { + global_proc()->internal_alloc_mtx.Lock(); + global_proc()->internal_alloc_mtx.Unlock(); +} + ScopedGlobalProcessor::ScopedGlobalProcessor() { GlobalProc *gp = global_proc(); ThreadState *thr = cur_thread(); @@ -110,6 +124,18 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() { gp->mtx.Unlock(); } +void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS { + global_proc()->mtx.Lock(); + global_proc()->internal_alloc_mtx.Lock(); + InternalAllocatorLock(); +} + +void AllocatorUnlock() NO_THREAD_SAFETY_ANALYSIS { + InternalAllocatorUnlock(); + global_proc()->internal_alloc_mtx.Unlock(); + global_proc()->mtx.Unlock(); +} + static constexpr uptr kMaxAllowedMallocSize = 1ull << 40; static uptr max_user_defined_malloc_size; @@ -342,6 +368,7 @@ void *Alloc(uptr sz) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } + InternalAllocAccess(); return InternalAlloc(sz, &thr->proc()->internal_alloc_cache); } @@ -351,6 +378,7 @@ void FreeImpl(void *p) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } + InternalAllocAccess(); InternalFree(p, &thr->proc()->internal_alloc_cache); } diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.h b/compiler-rt/lib/tsan/rtl/tsan_mman.h index efea5e5abdec..db8488eabbe2 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_mman.h +++ b/compiler-rt/lib/tsan/rtl/tsan_mman.h @@ -24,6 +24,8 @@ void ReplaceSystemMalloc(); void AllocatorProcStart(Processor *proc); void AllocatorProcFinish(Processor *proc); void AllocatorPrintStats(); +void AllocatorLock(); +void AllocatorUnlock(); // For user allocations. void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp index 3faa2d0c6192..1465f9953c19 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp @@ -25,6 +25,7 @@ #include "tsan_rtl.h" #include "tsan_flags.h" +#include <limits.h> #include <mach/mach.h> #include <pthread.h> #include <signal.h> @@ -45,70 +46,83 @@ namespace __tsan { #if !SANITIZER_GO -static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { - atomic_uintptr_t *a = (atomic_uintptr_t *)dst; - void *val = (void *)atomic_load_relaxed(a); - atomic_signal_fence(memory_order_acquire); // Turns the previous load into - // acquire wrt signals. - if (UNLIKELY(val == nullptr)) { - val = (void *)internal_mmap(nullptr, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); - CHECK(val); - void *cmp = nullptr; - if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val, - memory_order_acq_rel)) { - internal_munmap(val, size); - val = cmp; - } - } - return val; +static char main_thread_state[sizeof(ThreadState)] ALIGNED( + SANITIZER_CACHE_LINE_SIZE); +static ThreadState *dead_thread_state; +static pthread_key_t thread_state_key; + +// We rely on the following documented, but Darwin-specific behavior to keep the +// reference to the ThreadState object alive in TLS: +// pthread_key_create man page: +// If, after all the destructors have been called for all non-NULL values with +// associated destructors, there are still some non-NULL values with +// associated destructors, then the process is repeated. If, after at least +// [PTHREAD_DESTRUCTOR_ITERATIONS] iterations of destructor calls for +// outstanding non-NULL values, there are still some non-NULL values with +// associated destructors, the implementation stops calling destructors. +static_assert(PTHREAD_DESTRUCTOR_ITERATIONS == 4, "Small number of iterations"); +static void ThreadStateDestructor(void *thr) { + int res = pthread_setspecific(thread_state_key, thr); + CHECK_EQ(res, 0); } -// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is -// problematic, because there are several places where interceptors are called -// when TLVs are not accessible (early process startup, thread cleanup, ...). -// The following provides a "poor man's TLV" implementation, where we use the -// shadow memory of the pointer returned by pthread_self() to store a pointer to -// the ThreadState object. The main thread's ThreadState is stored separately -// in a static variable, because we need to access it even before the -// shadow memory is set up. -static uptr main_thread_identity = 0; -ALIGNED(64) static char main_thread_state[sizeof(ThreadState)]; -static ThreadState *main_thread_state_loc = (ThreadState *)main_thread_state; +static void InitializeThreadStateStorage() { + int res; + CHECK_EQ(thread_state_key, 0); + res = pthread_key_create(&thread_state_key, ThreadStateDestructor); + CHECK_EQ(res, 0); + res = pthread_setspecific(thread_state_key, main_thread_state); + CHECK_EQ(res, 0); -// We cannot use pthread_self() before libpthread has been initialized. Our -// current heuristic for guarding this is checking `main_thread_identity` which -// is only assigned in `__tsan::InitializePlatform`. -static ThreadState **cur_thread_location() { - if (main_thread_identity == 0) - return &main_thread_state_loc; - uptr thread_identity = (uptr)pthread_self(); - if (thread_identity == main_thread_identity) - return &main_thread_state_loc; - return (ThreadState **)MemToShadow(thread_identity); + auto dts = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState"); + dts->fast_state.SetIgnoreBit(); + dts->ignore_interceptors = 1; + dts->is_dead = true; + const_cast<Tid &>(dts->tid) = kInvalidTid; + res = internal_mprotect(dts, sizeof(ThreadState), PROT_READ); // immutable + CHECK_EQ(res, 0); + dead_thread_state = dts; } ThreadState *cur_thread() { - return (ThreadState *)SignalSafeGetOrAllocate( - (uptr *)cur_thread_location(), sizeof(ThreadState)); + // Some interceptors get called before libpthread has been initialized and in + // these cases we must avoid calling any pthread APIs. + if (UNLIKELY(!thread_state_key)) { + return (ThreadState *)main_thread_state; + } + + // We only reach this line after InitializeThreadStateStorage() ran, i.e, + // after TSan (and therefore libpthread) have been initialized. + ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key); + if (UNLIKELY(!thr)) { + thr = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState"); + int res = pthread_setspecific(thread_state_key, thr); + CHECK_EQ(res, 0); + } + return thr; } void set_cur_thread(ThreadState *thr) { - *cur_thread_location() = thr; + int res = pthread_setspecific(thread_state_key, thr); + CHECK_EQ(res, 0); } -// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call -// munmap first and then clear `fake_tls`; if we receive a signal in between, -// handler will try to access the unmapped ThreadState. void cur_thread_finalize() { - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == &main_thread_state_loc) { + ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key); + CHECK(thr); + if (thr == (ThreadState *)main_thread_state) { // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to // exit the main thread. Let's keep the main thread's ThreadState. return; } - internal_munmap(*thr_state_loc, sizeof(ThreadState)); - *thr_state_loc = nullptr; + // Intercepted functions can still get called after cur_thread_finalize() + // (called from DestroyThreadState()), so put a fake thread state for "dead" + // threads. An alternative solution would be to release the ThreadState + // object from THREAD_DESTROY (which is delivered later and on the parent + // thread) instead of THREAD_TERMINATE. + int res = pthread_setspecific(thread_state_key, dead_thread_state); + CHECK_EQ(res, 0); + UnmapOrDie(thr, sizeof(ThreadState)); } #endif @@ -222,11 +236,10 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { - if (thread == pthread_self()) { - ThreadState *thr = cur_thread(); - if (thr->tctx) { - DestroyThreadState(); - } + CHECK_EQ(thread, pthread_self()); + ThreadState *thr = cur_thread(); + if (thr->tctx) { + DestroyThreadState(); } } @@ -253,8 +266,7 @@ void InitializePlatform() { #if !SANITIZER_GO CheckAndProtect(); - CHECK_EQ(main_thread_identity, 0); - main_thread_identity = (uptr)pthread_self(); + InitializeThreadStateStorage(); prev_pthread_introspection_hook = pthread_introspection_hook_install(&my_pthread_introspection_hook); @@ -286,24 +298,11 @@ uptr ExtractLongJmpSp(uptr *env) { extern "C" void __tsan_tls_initialization() {} void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { - // The pointer to the ThreadState object is stored in the shadow memory - // of the tls. - uptr tls_end = tls_addr + tls_size; - uptr thread_identity = (uptr)pthread_self(); const uptr pc = StackTrace::GetNextInstructionPc( reinterpret_cast<uptr>(__tsan_tls_initialization)); - if (thread_identity == main_thread_identity) { - MemoryRangeImitateWrite(thr, pc, tls_addr, tls_size); - } else { - uptr thr_state_start = thread_identity; - uptr thr_state_end = thr_state_start + sizeof(uptr); - CHECK_GE(thr_state_start, tls_addr); - CHECK_LE(thr_state_start, tls_addr + tls_size); - CHECK_GE(thr_state_end, tls_addr); - CHECK_LE(thr_state_end, tls_addr + tls_size); - MemoryRangeImitateWrite(thr, pc, tls_addr, thr_state_start - tls_addr); - MemoryRangeImitateWrite(thr, pc, thr_state_end, tls_end - thr_state_end); - } + // Unlike Linux, we only store a pointer to the ThreadState object in TLS; + // just mark the entire range as written to. + MemoryRangeImitateWrite(thr, pc, tls_addr, tls_size); } #endif diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp index ff7726ef0608..c14af9788e32 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp @@ -34,6 +34,9 @@ extern "C" void __tsan_resume() { __tsan_resumed = 1; } +SANITIZER_WEAK_DEFAULT_IMPL +void __tsan_test_only_on_fork() {} + namespace __tsan { #if !SANITIZER_GO @@ -271,8 +274,39 @@ void DontNeedShadowFor(uptr addr, uptr size) { } #if !SANITIZER_GO +// We call UnmapShadow before the actual munmap, at that point we don't yet +// know if the provided address/size are sane. We can't call UnmapShadow +// after the actual munmap becuase at that point the memory range can +// already be reused for something else, so we can't rely on the munmap +// return value to understand is the values are sane. +// While calling munmap with insane values (non-canonical address, negative +// size, etc) is an error, the kernel won't crash. We must also try to not +// crash as the failure mode is very confusing (paging fault inside of the +// runtime on some derived shadow address). +static bool IsValidMmapRange(uptr addr, uptr size) { + if (size == 0) + return true; + if (static_cast<sptr>(size) < 0) + return false; + if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) + return false; + // Check that if the start of the region belongs to one of app ranges, + // end of the region belongs to the same region. + const uptr ranges[][2] = { + {LoAppMemBeg(), LoAppMemEnd()}, + {MidAppMemBeg(), MidAppMemEnd()}, + {HiAppMemBeg(), HiAppMemEnd()}, + }; + for (auto range : ranges) { + if (addr >= range[0] && addr < range[1]) + return addr + size <= range[1]; + } + return false; +} + void UnmapShadow(ThreadState *thr, uptr addr, uptr size) { - if (size == 0) return; + if (size == 0 || !IsValidMmapRange(addr, size)) + return; DontNeedShadowFor(addr, size); ScopedGlobalProcessor sgp; ctx->metamap.ResetRange(thr->proc(), addr, size); @@ -491,6 +525,7 @@ void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { ctx->thread_registry.Lock(); ctx->report_mtx.Lock(); ScopedErrorReportLock::Lock(); + AllocatorLock(); // Suppress all reports in the pthread_atfork callbacks. // Reports will deadlock on the report_mtx. // We could ignore sync operations as well, @@ -499,12 +534,20 @@ void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { thr->suppress_reports++; // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and // we'll assert in CheckNoLocks() unless we ignore interceptors. + // On OS X libSystem_atfork_prepare/parent/child callbacks are called + // after/before our callbacks and they call free. thr->ignore_interceptors++; + // Disables memory write in OnUserAlloc/Free. + thr->ignore_reads_and_writes++; + + __tsan_test_only_on_fork(); } void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { thr->suppress_reports--; // Enabled in ForkBefore. thr->ignore_interceptors--; + thr->ignore_reads_and_writes--; + AllocatorUnlock(); ScopedErrorReportLock::Unlock(); ctx->report_mtx.Unlock(); ctx->thread_registry.Unlock(); @@ -514,6 +557,8 @@ void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread) NO_THREAD_SAFETY_ANALYSIS { thr->suppress_reports--; // Enabled in ForkBefore. thr->ignore_interceptors--; + thr->ignore_reads_and_writes--; + AllocatorUnlock(); ScopedErrorReportLock::Unlock(); ctx->report_mtx.Unlock(); ctx->thread_registry.Unlock(); @@ -747,14 +792,17 @@ using namespace __tsan; MutexMeta mutex_meta[] = { {MutexInvalid, "Invalid", {}}, {MutexThreadRegistry, "ThreadRegistry", {}}, - {MutexTypeTrace, "Trace", {MutexLeaf}}, - {MutexTypeReport, "Report", {MutexTypeSyncVar}}, - {MutexTypeSyncVar, "SyncVar", {}}, + {MutexTypeTrace, "Trace", {}}, + {MutexTypeReport, + "Report", + {MutexTypeSyncVar, MutexTypeGlobalProc, MutexTypeTrace}}, + {MutexTypeSyncVar, "SyncVar", {MutexTypeTrace}}, {MutexTypeAnnotations, "Annotations", {}}, {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}}, {MutexTypeFired, "Fired", {MutexLeaf}}, {MutexTypeRacy, "Racy", {MutexLeaf}}, {MutexTypeGlobalProc, "GlobalProc", {}}, + {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}}, {}, }; diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp index 811695d144c5..f332a6a8d1d8 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp @@ -346,7 +346,7 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) { ThreadContext *tctx = FindThreadByTidLocked(b->tid); auto *loc = New<ReportLocation>(); loc->type = ReportLocationHeap; - loc->heap_chunk_start = (uptr)allocator()->GetBlockBegin((void *)addr); + loc->heap_chunk_start = block_begin; loc->heap_chunk_size = b->siz; loc->external_tag = b->tag; loc->tid = tctx ? tctx->tid : b->tid; |