diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2017-12-20 19:12:15 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2017-12-20 19:12:15 +0000 |
| commit | 8a51db9c42ac3e4a0d4c9bc2c4d61ccf0bd5f41a (patch) | |
| tree | ee959e7d66bf923526989c0e1a43cdb34958ec67 /contrib | |
| parent | acac075be8bce18f411c310a849e242ec445dfc3 (diff) | |
| parent | cdf4f3055e964bb585f294cf77cb549ead82783f (diff) | |
Notes
Diffstat (limited to 'contrib')
341 files changed, 14606 insertions, 4651 deletions
diff --git a/contrib/compiler-rt/include/sanitizer/allocator_interface.h b/contrib/compiler-rt/include/sanitizer/allocator_interface.h index 5220631619fc..2d35804b3764 100644 --- a/contrib/compiler-rt/include/sanitizer/allocator_interface.h +++ b/contrib/compiler-rt/include/sanitizer/allocator_interface.h @@ -76,6 +76,13 @@ extern "C" { void (*malloc_hook)(const volatile void *, size_t), void (*free_hook)(const volatile void *)); + /* Drains allocator quarantines (calling thread's and global ones), returns + freed memory back to OS and releases other non-essential internal allocator + resources in attempt to reduce process RSS. + Currently available with ASan only. + */ + void __sanitizer_purge_allocator(); + #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/asan_interface.h b/contrib/compiler-rt/include/sanitizer/asan_interface.h index 97ba0ceb0b23..e689a730e2c3 100644 --- a/contrib/compiler-rt/include/sanitizer/asan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/asan_interface.h @@ -144,6 +144,10 @@ extern "C" { void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg, void **end); + // Performs cleanup before a [[noreturn]] function. Must be called + // before things like _exit and execl to avoid false positives on stack. + void __asan_handle_no_return(void); + #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/coverage_interface.h b/contrib/compiler-rt/include/sanitizer/coverage_interface.h index 637379d47c41..081033cec8e5 100644 --- a/contrib/compiler-rt/include/sanitizer/coverage_interface.h +++ b/contrib/compiler-rt/include/sanitizer/coverage_interface.h @@ -22,8 +22,11 @@ extern "C" { // Record and dump coverage info. void __sanitizer_cov_dump(); - // Dump collected coverage info. Sorts pcs by module into individual - // .sancov files. + // Clear collected coverage info. + void __sanitizer_cov_reset(); + + // Dump collected coverage info. Sorts pcs by module into individual .sancov + // files. void __sanitizer_dump_coverage(const uintptr_t *pcs, uintptr_t len); #ifdef __cplusplus diff --git a/contrib/compiler-rt/include/sanitizer/hwasan_interface.h b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h new file mode 100644 index 000000000000..f70d87875968 --- /dev/null +++ b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h @@ -0,0 +1,33 @@ +//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Public interface header. +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_HWASAN_INTERFACE_H +#define SANITIZER_HWASAN_INTERFACE_H + +#include <sanitizer/common_interface_defs.h> + +#ifdef __cplusplus +extern "C" { +#endif + // This function may be optionally provided by user and should return + // a string containing HWASan runtime options. See asan_flags.h for details. + const char* __hwasan_default_options(); + + void __hwasan_enable_allocator_tagging(); + void __hwasan_disable_allocator_tagging(); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_HWASAN_INTERFACE_H diff --git a/contrib/compiler-rt/include/sanitizer/lsan_interface.h b/contrib/compiler-rt/include/sanitizer/lsan_interface.h index 8fb8e756da26..e3e509ff7d9f 100644 --- a/contrib/compiler-rt/include/sanitizer/lsan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/lsan_interface.h @@ -64,8 +64,14 @@ extern "C" { // for the program it is linked into (if the return value is non-zero). This // function must be defined as returning a constant value; any behavior beyond // that is unsupported. + // To avoid dead stripping, you may need to define this function with + // __attribute__((used)) int __lsan_is_turned_off(); + // This function may be optionally provided by user and should return + // a string containing LSan runtime options. See lsan_flags.inc for details. + const char *__lsan_default_options(); + // This function may be optionally provided by the user and should return // a string containing LSan suppressions. const char *__lsan_default_suppressions(); diff --git a/contrib/compiler-rt/include/sanitizer/scudo_interface.h b/contrib/compiler-rt/include/sanitizer/scudo_interface.h new file mode 100644 index 000000000000..cab7e0f5f344 --- /dev/null +++ b/contrib/compiler-rt/include/sanitizer/scudo_interface.h @@ -0,0 +1,34 @@ +//===-- sanitizer/scudo_interface.h -----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// Public Scudo interface header. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_SCUDO_INTERFACE_H_ +#define SANITIZER_SCUDO_INTERFACE_H_ + +#include <sanitizer/common_interface_defs.h> + +#ifdef __cplusplus +extern "C" { +#endif + // This function may be optionally provided by a user and should return + // a string containing Scudo runtime options. See scudo_flags.h for details. + const char* __scudo_default_options(); + + // This function allows to set the RSS limit at runtime. This can be either + // the hard limit (HardLimit=1) or the soft limit (HardLimit=0). The limit + // can be removed by setting LimitMb to 0. This function's parameters should + // be fully trusted to avoid security mishaps. + void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_SCUDO_INTERFACE_H_ diff --git a/contrib/compiler-rt/include/sanitizer/tsan_interface.h b/contrib/compiler-rt/include/sanitizer/tsan_interface.h index 5ea09ab5c1ff..530885014ac3 100644 --- a/contrib/compiler-rt/include/sanitizer/tsan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/tsan_interface.h @@ -44,6 +44,11 @@ const unsigned __tsan_mutex_linker_init = 1 << 0; const unsigned __tsan_mutex_write_reentrant = 1 << 1; // Mutex is read reentrant. const unsigned __tsan_mutex_read_reentrant = 1 << 2; +// Mutex does not have static storage duration, and must not be used after +// its destructor runs. The opposite of __tsan_mutex_linker_init. +// If this flag is passed to __tsan_mutex_destroy, then the destruction +// is ignored unless this flag was previously set on the mutex. +const unsigned __tsan_mutex_not_static = 1 << 8; // Mutex operation flags: @@ -70,6 +75,7 @@ void __tsan_mutex_create(void *addr, unsigned flags); // Annotate destruction of a mutex. // Supported flags: // - __tsan_mutex_linker_init +// - __tsan_mutex_not_static void __tsan_mutex_destroy(void *addr, unsigned flags); // Annotate start of lock operation. diff --git a/contrib/compiler-rt/include/xray/xray_interface.h b/contrib/compiler-rt/include/xray/xray_interface.h index 564613417069..d08039a67fa2 100644 --- a/contrib/compiler-rt/include/xray/xray_interface.h +++ b/contrib/compiler-rt/include/xray/xray_interface.h @@ -106,6 +106,14 @@ extern uintptr_t __xray_function_address(int32_t FuncId); /// encounter errors (when there are no instrumented functions, etc.). extern size_t __xray_max_function_id(); +/// Initialize the required XRay data structures. This is useful in cases where +/// users want to control precisely when the XRay instrumentation data +/// structures are initialized, for example when the XRay library is built with +/// the XRAY_NO_PREINIT preprocessor definition. +/// +/// Calling __xray_init() more than once is safe across multiple threads. +extern void __xray_init(); + } // end extern "C" #endif // XRAY_XRAY_INTERFACE_H diff --git a/contrib/compiler-rt/include/xray/xray_log_interface.h b/contrib/compiler-rt/include/xray/xray_log_interface.h index cdb20094dc26..6c53cdaebfe2 100644 --- a/contrib/compiler-rt/include/xray/xray_log_interface.h +++ b/contrib/compiler-rt/include/xray/xray_log_interface.h @@ -128,6 +128,16 @@ enum XRayLogFlushStatus { XRAY_LOG_FLUSHED = 2, }; +/// This enum indicates the installation state of a logging implementation, when +/// associating a mode to a particular logging implementation through +/// `__xray_log_register_impl(...)` or through `__xray_log_select_mode(...`. +enum XRayLogRegisterStatus { + XRAY_REGISTRATION_OK = 0, + XRAY_DUPLICATE_MODE = 1, + XRAY_MODE_NOT_FOUND = 2, + XRAY_INCOMPLETE_IMPL = 3, +}; + /// A valid XRay logging implementation MUST provide all of the function /// pointers in XRayLogImpl when being installed through `__xray_set_log_impl`. /// To be precise, ALL the functions pointers MUST NOT be nullptr. @@ -159,6 +169,9 @@ struct XRayLogImpl { /// always have a handler for function entry and exit events. In case the /// implementation wants to support arg1 (or other future extensions to XRay /// logging) those MUST be installed by the installed 'log_init' handler. + /// + /// Because we didn't want to change the ABI of this struct, the arg1 handler + /// may be silently overwritten during initialization as well. void (*handle_arg0)(int32_t, XRayEntryType); /// The log implementation provided routine for when __xray_log_flushLog() is @@ -186,6 +199,34 @@ struct XRayLogImpl { /// called while in any other states. void __xray_set_log_impl(XRayLogImpl Impl); +/// This function registers a logging implementation against a "mode" +/// identifier. This allows multiple modes to be registered, and chosen at +/// runtime using the same mode identifier through +/// `__xray_log_select_mode(...)`. +/// +/// We treat the Mode identifier as a null-terminated byte string, as the +/// identifier used when retrieving the log impl. +/// +/// Returns: +/// - XRAY_REGISTRATION_OK on success. +/// - XRAY_DUPLICATE_MODE when an implementation is already associated with +/// the provided Mode; does not update the already-registered +/// implementation. +XRayLogRegisterStatus __xray_log_register_mode(const char *Mode, + XRayLogImpl Impl); + +/// This function selects the implementation associated with Mode that has been +/// registered through __xray_log_register_mode(...) and installs that +/// implementation (as if through calling __xray_set_log_impl(...)). The same +/// caveats apply to __xray_log_select_mode(...) as with +/// __xray_log_set_log_impl(...). +/// +/// Returns: +/// - XRAY_REGISTRATION_OK on success. +/// - XRAY_MODE_NOT_FOUND if there is no implementation associated with Mode; +/// does not update the currently installed implementation. +XRayLogRegisterStatus __xray_log_select_mode(const char *Mode); + /// This function removes the currently installed implementation. It will also /// uninstall any handlers that have been previously installed. It does NOT /// unpatch the instrumentation sleds. @@ -220,12 +261,19 @@ XRayLogFlushStatus __xray_log_flushLog(); namespace __xray { -// Options used by the LLVM XRay FDR implementation. +/// Options used by the LLVM XRay FDR logging implementation. struct FDRLoggingOptions { bool ReportErrors = false; int Fd = -1; }; +/// Options used by the LLVM XRay Basic (Naive) logging implementation. +struct BasicLoggingOptions { + int DurationFilterMicros = 0; + size_t MaxStackDepth = 0; + size_t ThreadBufferSize = 0; +}; + } // namespace __xray #endif // XRAY_XRAY_LOG_INTERFACE_H diff --git a/contrib/compiler-rt/include/xray/xray_records.h b/contrib/compiler-rt/include/xray/xray_records.h index feb8d228b2fd..d4b7b4c31a9a 100644 --- a/contrib/compiler-rt/include/xray/xray_records.h +++ b/contrib/compiler-rt/include/xray/xray_records.h @@ -17,6 +17,8 @@ #ifndef XRAY_XRAY_RECORDS_H #define XRAY_XRAY_RECORDS_H +#include <cstdint> + namespace __xray { enum FileTypes { @@ -65,18 +67,23 @@ static_assert(sizeof(XRayFileHeader) == 32, "XRayFileHeader != 32 bytes"); enum RecordTypes { NORMAL = 0, + ARG_PAYLOAD = 1, }; struct alignas(32) XRayRecord { // This is the type of the record being written. We use 16 bits to allow us to // treat this as a discriminant, and so that the first 4 bytes get packed // properly. See RecordTypes for more supported types. - uint16_t RecordType = 0; + uint16_t RecordType = RecordTypes::NORMAL; // The CPU where the thread is running. We assume number of CPUs <= 256. uint8_t CPU = 0; - // The type of the event. Usually either ENTER = 0 or EXIT = 1. + // The type of the event. One of the following: + // ENTER = 0 + // EXIT = 1 + // TAIL_EXIT = 2 + // ENTER_ARG = 3 uint8_t Type = 0; // The function ID for the record. @@ -94,6 +101,32 @@ struct alignas(32) XRayRecord { static_assert(sizeof(XRayRecord) == 32, "XRayRecord != 32 bytes"); +struct alignas(32) XRayArgPayload { + // We use the same 16 bits as a discriminant for the records in the log here + // too, and so that the first 4 bytes are packed properly. + uint16_t RecordType = RecordTypes::ARG_PAYLOAD; + + // Add a few bytes to pad. + uint8_t Padding[2] = {}; + + // The function ID for the record. + int32_t FuncId = 0; + + // The thread ID for the currently running thread. + uint32_t TId = 0; + + // Add more padding. + uint8_t Padding2[4] = {}; + + // The argument payload. + uint64_t Arg = 0; + + // The rest of this record ought to be left as padding. + uint8_t TailPadding[8] = {}; +} __attribute__((packed)); + +static_assert(sizeof(XRayArgPayload) == 32, "XRayArgPayload != 32 bytes"); + } // namespace __xray #endif // XRAY_XRAY_RECORDS_H diff --git a/contrib/compiler-rt/lib/BlocksRuntime/Block.h b/contrib/compiler-rt/lib/BlocksRuntime/Block.h index 55cdd01a9123..e6cb142fd755 100644 --- a/contrib/compiler-rt/lib/BlocksRuntime/Block.h +++ b/contrib/compiler-rt/lib/BlocksRuntime/Block.h @@ -27,7 +27,7 @@ #if !defined(BLOCK_EXPORT) # if defined(__cplusplus) -# define BLOCK_EXPORT extern "C" +# define BLOCK_EXPORT extern "C" # else # define BLOCK_EXPORT extern # endif diff --git a/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h b/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h index 8ae821815ebe..91d8d8a9850b 100644 --- a/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h +++ b/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h @@ -27,7 +27,7 @@ #if !defined(BLOCK_EXPORT) # if defined(__cplusplus) -# define BLOCK_EXPORT extern "C" +# define BLOCK_EXPORT extern "C" # else # define BLOCK_EXPORT extern # endif diff --git a/contrib/compiler-rt/lib/asan/asan_activation.cc b/contrib/compiler-rt/lib/asan/asan_activation.cc index 66eba9ce2748..d642be93488d 100644 --- a/contrib/compiler-rt/lib/asan/asan_activation.cc +++ b/contrib/compiler-rt/lib/asan/asan_activation.cc @@ -16,8 +16,10 @@ #include "asan_allocator.h" #include "asan_flags.h" #include "asan_internal.h" +#include "asan_mapping.h" #include "asan_poisoning.h" #include "asan_stack.h" +#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" namespace __asan { @@ -110,8 +112,9 @@ void AsanDeactivate() { AllocatorOptions disabled = asan_deactivated_flags.allocator_options; disabled.quarantine_size_mb = 0; disabled.thread_local_quarantine_size_kb = 0; - disabled.min_redzone = 16; // Redzone must be at least 16 bytes long. - disabled.max_redzone = 16; + // Redzone must be at least Max(16, granularity) bytes long. + disabled.min_redzone = Max(16, (int)SHADOW_GRANULARITY); + disabled.max_redzone = disabled.min_redzone; disabled.alloc_dealloc_mismatch = false; disabled.may_return_null = true; ReInitializeAllocator(disabled); diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.cc b/contrib/compiler-rt/lib/asan/asan_allocator.cc index 92963ddfc4da..a437ae1cd3b1 100644 --- a/contrib/compiler-rt/lib/asan/asan_allocator.cc +++ b/contrib/compiler-rt/lib/asan/asan_allocator.cc @@ -84,7 +84,10 @@ struct ChunkHeader { // This field is used for small sizes. For large sizes it is equal to // SizeClassMap::kMaxSize and the actual size is stored in the // SecondaryAllocator's metadata. - u32 user_requested_size; + u32 user_requested_size : 29; + // align < 8 -> 0 + // else -> log2(min(align, 512)) - 2 + u32 user_requested_alignment_log : 3; u32 alloc_context_id; }; @@ -271,9 +274,9 @@ struct Allocator { atomic_store(&max_redzone, options.max_redzone, memory_order_release); } - void Initialize(const AllocatorOptions &options) { + void InitLinkerInitialized(const AllocatorOptions &options) { SetAllocatorMayReturnNull(options.may_return_null); - allocator.Init(options.release_to_os_interval_ms); + allocator.InitLinkerInitialized(options.release_to_os_interval_ms); SharedInitCode(options); } @@ -351,6 +354,20 @@ struct Allocator { return Min(Max(rz_log, RZSize2Log(min_rz)), RZSize2Log(max_rz)); } + static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) { + if (user_requested_alignment < 8) + return 0; + if (user_requested_alignment > 512) + user_requested_alignment = 512; + return Log2(user_requested_alignment) - 2; + } + + static uptr ComputeUserAlignment(uptr user_requested_alignment_log) { + if (user_requested_alignment_log == 0) + return 0; + return 1LL << (user_requested_alignment_log + 2); + } + // We have an address between two chunks, and we want to report just one. AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk, AsanChunk *right_chunk) { @@ -385,6 +402,8 @@ struct Allocator { Flags &fl = *flags(); CHECK(stack); const uptr min_alignment = SHADOW_GRANULARITY; + const uptr user_requested_alignment_log = + ComputeUserRequestedAlignmentLog(alignment); if (alignment < min_alignment) alignment = min_alignment; if (size == 0) { @@ -472,6 +491,7 @@ struct Allocator { meta[0] = size; meta[1] = chunk_beg; } + m->user_requested_alignment_log = user_requested_alignment_log; m->alloc_context_id = StackDepotPut(*stack); @@ -573,8 +593,8 @@ struct Allocator { } } - void Deallocate(void *ptr, uptr delete_size, BufferedStackTrace *stack, - AllocType alloc_type) { + void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment, + BufferedStackTrace *stack, AllocType alloc_type) { uptr p = reinterpret_cast<uptr>(ptr); if (p == 0) return; @@ -601,11 +621,14 @@ struct Allocator { ReportAllocTypeMismatch((uptr)ptr, stack, (AllocType)m->alloc_type, (AllocType)alloc_type); } - } - - if (delete_size && flags()->new_delete_type_mismatch && - delete_size != m->UsedSize()) { - ReportNewDeleteSizeMismatch(p, delete_size, stack); + } else { + if (flags()->new_delete_type_mismatch && + (alloc_type == FROM_NEW || alloc_type == FROM_NEW_BR) && + ((delete_size && delete_size != m->UsedSize()) || + ComputeUserRequestedAlignmentLog(delete_alignment) != + m->user_requested_alignment_log)) { + ReportNewDeleteTypeMismatch(p, delete_size, delete_alignment, stack); + } } QuarantineChunk(m, ptr, stack); @@ -631,7 +654,7 @@ struct Allocator { // If realloc() races with free(), we may start copying freed memory. // However, we will report racy double-free later anyway. REAL(memcpy)(new_ptr, old_ptr, memcpy_size); - Deallocate(old_ptr, 0, stack, FROM_MALLOC); + Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC); } return new_ptr; } @@ -716,6 +739,22 @@ struct Allocator { return AsanChunkView(m1); } + void Purge() { + AsanThread *t = GetCurrentThread(); + if (t) { + AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); + quarantine.DrainAndRecycle(GetQuarantineCache(ms), + QuarantineCallback(GetAllocatorCache(ms))); + } + { + SpinMutexLock l(&fallback_mutex); + quarantine.DrainAndRecycle(&fallback_quarantine_cache, + QuarantineCallback(&fallback_allocator_cache)); + } + + allocator.ForceReleaseToOS(); + } + void PrintStats() { allocator.PrintStats(); quarantine.PrintStats(); @@ -750,6 +789,9 @@ bool AsanChunkView::IsQuarantined() const { uptr AsanChunkView::Beg() const { return chunk_->Beg(); } uptr AsanChunkView::End() const { return Beg() + UsedSize(); } uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); } +u32 AsanChunkView::UserRequestedAlignment() const { + return Allocator::ComputeUserAlignment(chunk_->user_requested_alignment_log); +} uptr AsanChunkView::AllocTid() const { return chunk_->alloc_tid; } uptr AsanChunkView::FreeTid() const { return chunk_->free_tid; } AllocType AsanChunkView::GetAllocType() const { @@ -775,7 +817,7 @@ StackTrace AsanChunkView::GetFreeStack() const { } void InitializeAllocator(const AllocatorOptions &options) { - instance.Initialize(options); + instance.InitLinkerInitialized(options); } void ReInitializeAllocator(const AllocatorOptions &options) { @@ -802,12 +844,12 @@ void PrintInternalAllocatorStats() { } void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { - instance.Deallocate(ptr, 0, stack, alloc_type); + instance.Deallocate(ptr, 0, 0, stack, alloc_type); } -void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack, - AllocType alloc_type) { - instance.Deallocate(ptr, size, stack, alloc_type); +void asan_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type) { + instance.Deallocate(ptr, size, alignment, stack, alloc_type); } void *asan_malloc(uptr size, BufferedStackTrace *stack) { @@ -823,7 +865,7 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true)); if (size == 0) { if (flags()->allocator_frees_and_returns_null_on_realloc_zero) { - instance.Deallocate(p, 0, stack, FROM_MALLOC); + instance.Deallocate(p, 0, 0, stack, FROM_MALLOC); return nullptr; } // Allocate a size of 1 if we shouldn't free() on Realloc to 0 @@ -839,6 +881,10 @@ void *asan_valloc(uptr size, BufferedStackTrace *stack) { void *asan_pvalloc(uptr size, BufferedStackTrace *stack) { uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + return AsanAllocator::FailureHandler::OnBadRequest(); + } // pvalloc(0) should allocate one page. size = size ? RoundUpTo(size, PageSize) : PageSize; return SetErrnoOnNull( @@ -1007,6 +1053,10 @@ uptr __sanitizer_get_allocated_size(const void *p) { return allocated_size; } +void __sanitizer_purge_allocator() { + instance.Purge(); +} + #if !SANITIZER_SUPPORTS_WEAK_HOOKS // Provide default (no-op) implementation of malloc hooks. SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.h b/contrib/compiler-rt/lib/asan/asan_allocator.h index ad1aeb58a86b..26483db4c60e 100644 --- a/contrib/compiler-rt/lib/asan/asan_allocator.h +++ b/contrib/compiler-rt/lib/asan/asan_allocator.h @@ -58,6 +58,7 @@ class AsanChunkView { uptr Beg() const; // First byte of user memory. uptr End() const; // Last byte of user memory. uptr UsedSize() const; // Size requested by the user. + u32 UserRequestedAlignment() const; // Originally requested alignment. uptr AllocTid() const; uptr FreeTid() const; bool Eq(const AsanChunkView &c) const { return chunk_ == c.chunk_; } @@ -119,7 +120,11 @@ struct AsanMapUnmapCallback { }; #if SANITIZER_CAN_USE_ALLOCATOR64 -# if defined(__powerpc64__) +# if SANITIZER_FUCHSIA +const uptr kAllocatorSpace = ~(uptr)0; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +typedef DefaultSizeClassMap SizeClassMap; +# elif defined(__powerpc64__) const uptr kAllocatorSpace = 0xa0000000000ULL; const uptr kAllocatorSize = 0x20000000000ULL; // 2T. typedef DefaultSizeClassMap SizeClassMap; @@ -193,8 +198,8 @@ struct AsanThreadLocalMallocStorage { void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, AllocType alloc_type); void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type); -void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack, - AllocType alloc_type); +void asan_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type); void *asan_malloc(uptr size, BufferedStackTrace *stack); void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack); diff --git a/contrib/compiler-rt/lib/asan/asan_descriptions.cc b/contrib/compiler-rt/lib/asan/asan_descriptions.cc index 822a6a62d64e..86c6af7d9f75 100644 --- a/contrib/compiler-rt/lib/asan/asan_descriptions.cc +++ b/contrib/compiler-rt/lib/asan/asan_descriptions.cc @@ -122,6 +122,7 @@ static void GetAccessToHeapChunkInformation(ChunkAccess *descr, } descr->chunk_begin = chunk.Beg(); descr->chunk_size = chunk.UsedSize(); + descr->user_requested_alignment = chunk.UserRequestedAlignment(); descr->alloc_type = chunk.GetAllocType(); } @@ -150,7 +151,7 @@ static void PrintHeapChunkAccess(uptr addr, const ChunkAccess &descr) { str.append(" %zu-byte region [%p,%p)\n", descr.chunk_size, (void *)descr.chunk_begin, (void *)(descr.chunk_begin + descr.chunk_size)); - str.append("%s", d.EndLocation()); + str.append("%s", d.Default()); Printf("%s", str.data()); } @@ -260,7 +261,7 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr, // FIXME: we may want to also print the size of the access here, // but in case of accesses generated by memset it may be confusing. str.append("%s <== Memory access at offset %zd %s this variable%s\n", - d.Location(), addr, pos_descr, d.EndLocation()); + d.Location(), addr, pos_descr, d.Default()); } else { str.append("\n"); } @@ -295,7 +296,7 @@ static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size, MaybeDemangleGlobalName(g.name)); PrintGlobalLocation(&str, g); str.append("' (0x%zx) of size %zu\n", g.beg, g.size); - str.append("%s", d.EndLocation()); + str.append("%s", d.Default()); PrintGlobalNameIfASCII(&str, g); Printf("%s", str.data()); } @@ -335,6 +336,26 @@ void GlobalAddressDescription::Print(const char *bug_type) const { } } +bool GlobalAddressDescription::PointsInsideTheSameVariable( + const GlobalAddressDescription &other) const { + if (size == 0 || other.size == 0) return false; + + for (uptr i = 0; i < size; i++) { + const __asan_global &a = globals[i]; + for (uptr j = 0; j < other.size; j++) { + const __asan_global &b = other.globals[j]; + if (a.beg == b.beg && + a.beg <= addr && + b.beg <= other.addr && + (addr + access_size) < (a.beg + a.size) && + (other.addr + other.access_size) < (b.beg + b.size)) + return true; + } + } + + return false; +} + void StackAddressDescription::Print() const { Decorator d; char tname[128]; @@ -343,10 +364,10 @@ void StackAddressDescription::Print() const { ThreadNameWithParenthesis(tid, tname, sizeof(tname))); if (!frame_descr) { - Printf("%s\n", d.EndLocation()); + Printf("%s\n", d.Default()); return; } - Printf(" at offset %zu in frame%s\n", offset, d.EndLocation()); + Printf(" at offset %zu in frame%s\n", offset, d.Default()); // Now we print the frame where the alloca has happened. // We print this frame as a stack trace with one element. @@ -355,7 +376,7 @@ void StackAddressDescription::Print() const { // previously. That's unfortunate, but I have no better solution, // especially given that the alloca may be from entirely different place // (e.g. use-after-scope, or different thread's stack). - Printf("%s", d.EndLocation()); + Printf("%s", d.Default()); StackTrace alloca_stack(&frame_pc, 1); alloca_stack.Print(); @@ -405,18 +426,18 @@ void HeapAddressDescription::Print() const { Printf("%sfreed by thread T%d%s here:%s\n", d.Allocation(), free_thread->tid, ThreadNameWithParenthesis(free_thread, tname, sizeof(tname)), - d.EndAllocation()); + d.Default()); StackTrace free_stack = GetStackTraceFromId(free_stack_id); free_stack.Print(); Printf("%spreviously allocated by thread T%d%s here:%s\n", d.Allocation(), alloc_thread->tid, ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)), - d.EndAllocation()); + d.Default()); } else { Printf("%sallocated by thread T%d%s here:%s\n", d.Allocation(), alloc_thread->tid, ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)), - d.EndAllocation()); + d.Default()); } alloc_stack.Print(); DescribeThread(GetCurrentThread()); diff --git a/contrib/compiler-rt/lib/asan/asan_descriptions.h b/contrib/compiler-rt/lib/asan/asan_descriptions.h index 0ee677eb7d0e..1a1b01cf20cd 100644 --- a/contrib/compiler-rt/lib/asan/asan_descriptions.h +++ b/contrib/compiler-rt/lib/asan/asan_descriptions.h @@ -34,11 +34,8 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() {} const char *Access() { return Blue(); } - const char *EndAccess() { return Default(); } const char *Location() { return Green(); } - const char *EndLocation() { return Default(); } const char *Allocation() { return Magenta(); } - const char *EndAllocation() { return Default(); } const char *ShadowByte(u8 byte) { switch (byte) { @@ -72,9 +69,6 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator { return Default(); } } - const char *EndShadowByte() { return Default(); } - const char *MemoryByte() { return Magenta(); } - const char *EndMemoryByte() { return Default(); } }; enum ShadowKind : u8 { @@ -108,6 +102,7 @@ struct ChunkAccess { sptr offset; uptr chunk_begin; uptr chunk_size; + u32 user_requested_alignment : 12; u32 access_type : 2; u32 alloc_type : 2; }; @@ -151,6 +146,10 @@ struct GlobalAddressDescription { u8 size; void Print(const char *bug_type = "") const; + + // Returns true when this descriptions points inside the same global variable + // as other. Descriptions can have different address within the variable + bool PointsInsideTheSameVariable(const GlobalAddressDescription &other) const; }; bool GetGlobalAddressInformation(uptr addr, uptr access_size, diff --git a/contrib/compiler-rt/lib/asan/asan_errors.cc b/contrib/compiler-rt/lib/asan/asan_errors.cc index b7a38eb7cece..0f4a3abff9a7 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.cc +++ b/contrib/compiler-rt/lib/asan/asan_errors.cc @@ -13,7 +13,6 @@ //===----------------------------------------------------------------------===// #include "asan_errors.h" -#include <signal.h> #include "asan_descriptions.h" #include "asan_mapping.h" #include "asan_report.h" @@ -22,82 +21,26 @@ namespace __asan { -void ErrorStackOverflow::Print() { - Decorator d; - Printf("%s", d.Warning()); - Report( - "ERROR: AddressSanitizer: %s on address %p" - " (pc %p bp %p sp %p T%d)\n", scariness.GetDescription(), - (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid); - Printf("%s", d.EndWarning()); - scariness.Print(); - BufferedStackTrace stack; - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context, - common_flags()->fast_unwind_on_fatal); - stack.Print(); - ReportErrorSummary(scariness.GetDescription(), &stack); -} - -static void MaybeDumpInstructionBytes(uptr pc) { - if (!flags()->dump_instruction_bytes || (pc < GetPageSizeCached())) return; - InternalScopedString str(1024); - str.append("First 16 instruction bytes at pc: "); - if (IsAccessibleMemoryRange(pc, 16)) { - for (int i = 0; i < 16; ++i) { - PrintMemoryByte(&str, "", ((u8 *)pc)[i], /*in_shadow*/ false, " "); - } - str.append("\n"); - } else { - str.append("unaccessible\n"); - } - Report("%s", str.data()); -} - -static void MaybeDumpRegisters(void *context) { - if (!flags()->dump_registers) return; - SignalContext::DumpAllRegisters(context); -} - -static void MaybeReportNonExecRegion(uptr pc) { -#if SANITIZER_FREEBSD || SANITIZER_LINUX - MemoryMappingLayout proc_maps(/*cache_enabled*/ true); - MemoryMappedSegment segment; - while (proc_maps.Next(&segment)) { - if (pc >= segment.start && pc < segment.end && !segment.IsExecutable()) - Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n"); - } +static void OnStackUnwind(const SignalContext &sig, + const void *callback_context, + BufferedStackTrace *stack) { + bool fast = common_flags()->fast_unwind_on_fatal; +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace() + // yields the call stack of the signal's handler and not of the code + // that raised the signal (as it does on Linux). + fast = true; #endif + // Tests and maybe some users expect that scariness is going to be printed + // just before the stack. As only asan has scariness score we have no + // corresponding code in the sanitizer_common and we use this callback to + // print it. + static_cast<const ScarinessScoreBase *>(callback_context)->Print(); + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, fast); } void ErrorDeadlySignal::Print() { - Decorator d; - Printf("%s", d.Warning()); - const char *description = __sanitizer::DescribeSignalOrException(signo); - Report( - "ERROR: AddressSanitizer: %s on unknown address %p (pc %p bp %p sp %p " - "T%d)\n", - description, (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid); - Printf("%s", d.EndWarning()); - if (pc < GetPageSizeCached()) Report("Hint: pc points to the zero page.\n"); - if (is_memory_access) { - const char *access_type = - write_flag == SignalContext::WRITE - ? "WRITE" - : (write_flag == SignalContext::READ ? "READ" : "UNKNOWN"); - Report("The signal is caused by a %s memory access.\n", access_type); - if (addr < GetPageSizeCached()) - Report("Hint: address points to the zero page.\n"); - } - MaybeReportNonExecRegion(pc); - scariness.Print(); - BufferedStackTrace stack; - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context, - common_flags()->fast_unwind_on_fatal); - stack.Print(); - MaybeDumpInstructionBytes(pc); - MaybeDumpRegisters(context); - Printf("AddressSanitizer can not provide additional info.\n"); - ReportErrorSummary(description, &stack); + ReportDeadlySignal(signal, tid, &OnStackUnwind, &scariness); } void ErrorDoubleFree::Print() { @@ -109,7 +52,7 @@ void ErrorDoubleFree::Print() { "thread T%d%s:\n", scariness.GetDescription(), addr_description.addr, tid, ThreadNameWithParenthesis(tid, tname, sizeof(tname))); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); scariness.Print(); GET_STACK_TRACE_FATAL(second_free_stack->trace[0], second_free_stack->top_frame_bp); @@ -118,7 +61,7 @@ void ErrorDoubleFree::Print() { ReportErrorSummary(scariness.GetDescription(), &stack); } -void ErrorNewDeleteSizeMismatch::Print() { +void ErrorNewDeleteTypeMismatch::Print() { Decorator d; Printf("%s", d.Warning()); char tname[128]; @@ -127,11 +70,29 @@ void ErrorNewDeleteSizeMismatch::Print() { "T%d%s:\n", scariness.GetDescription(), addr_description.addr, tid, ThreadNameWithParenthesis(tid, tname, sizeof(tname))); - Printf("%s object passed to delete has wrong type:\n", d.EndWarning()); - Printf( - " size of the allocated type: %zd bytes;\n" - " size of the deallocated type: %zd bytes.\n", - addr_description.chunk_access.chunk_size, delete_size); + Printf("%s object passed to delete has wrong type:\n", d.Default()); + if (delete_size != 0) { + Printf( + " size of the allocated type: %zd bytes;\n" + " size of the deallocated type: %zd bytes.\n", + addr_description.chunk_access.chunk_size, delete_size); + } + const uptr user_alignment = + addr_description.chunk_access.user_requested_alignment; + if (delete_alignment != user_alignment) { + char user_alignment_str[32]; + char delete_alignment_str[32]; + internal_snprintf(user_alignment_str, sizeof(user_alignment_str), + "%zd bytes", user_alignment); + internal_snprintf(delete_alignment_str, sizeof(delete_alignment_str), + "%zd bytes", delete_alignment); + static const char *kDefaultAlignment = "default-aligned"; + Printf( + " alignment of the allocated type: %s;\n" + " alignment of the deallocated type: %s.\n", + user_alignment > 0 ? user_alignment_str : kDefaultAlignment, + delete_alignment > 0 ? delete_alignment_str : kDefaultAlignment); + } CHECK_GT(free_stack->size, 0); scariness.Print(); GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp); @@ -152,7 +113,7 @@ void ErrorFreeNotMalloced::Print() { "which was not malloc()-ed: %p in thread T%d%s\n", addr_description.Address(), tid, ThreadNameWithParenthesis(tid, tname, sizeof(tname))); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); CHECK_GT(free_stack->size, 0); scariness.Print(); GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp); @@ -173,7 +134,7 @@ void ErrorAllocTypeMismatch::Print() { scariness.GetDescription(), alloc_names[alloc_type], dealloc_names[dealloc_type], addr_description.addr); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); CHECK_GT(dealloc_stack->size, 0); scariness.Print(); GET_STACK_TRACE_FATAL(dealloc_stack->trace[0], dealloc_stack->top_frame_bp); @@ -192,7 +153,7 @@ void ErrorMallocUsableSizeNotOwned::Print() { "ERROR: AddressSanitizer: attempting to call malloc_usable_size() for " "pointer which is not owned: %p\n", addr_description.Address()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); stack->Print(); addr_description.Print(); ReportErrorSummary(scariness.GetDescription(), stack); @@ -205,7 +166,7 @@ void ErrorSanitizerGetAllocatedSizeNotOwned::Print() { "ERROR: AddressSanitizer: attempting to call " "__sanitizer_get_allocated_size() for pointer which is not owned: %p\n", addr_description.Address()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); stack->Print(); addr_description.Print(); ReportErrorSummary(scariness.GetDescription(), stack); @@ -222,7 +183,7 @@ void ErrorStringFunctionMemoryRangesOverlap::Print() { bug_type, addr1_description.Address(), addr1_description.Address() + length1, addr2_description.Address(), addr2_description.Address() + length2); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); scariness.Print(); stack->Print(); addr1_description.Print(); @@ -235,7 +196,7 @@ void ErrorStringFunctionSizeOverflow::Print() { Printf("%s", d.Warning()); Report("ERROR: AddressSanitizer: %s: (size=%zd)\n", scariness.GetDescription(), size); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); scariness.Print(); stack->Print(); addr_description.Print(); @@ -263,7 +224,7 @@ void ErrorODRViolation::Print() { Printf("%s", d.Warning()); Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(), global1.beg); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); InternalScopedString g1_loc(256), g2_loc(256); PrintGlobalLocation(&g1_loc, global1); PrintGlobalLocation(&g2_loc, global2); @@ -292,7 +253,7 @@ void ErrorInvalidPointerPair::Print() { Printf("%s", d.Warning()); Report("ERROR: AddressSanitizer: %s: %p %p\n", scariness.GetDescription(), addr1_description.Address(), addr2_description.Address()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); GET_STACK_TRACE_FATAL(pc, bp); stack.Print(); addr1_description.Print(); @@ -477,9 +438,14 @@ static void PrintShadowMemoryForAddress(uptr addr) { InternalScopedString str(4096 * 8); str.append("Shadow bytes around the buggy address:\n"); for (int i = -5; i <= 5; i++) { + uptr row_shadow_addr = aligned_shadow + i * n_bytes_per_row; + // Skip rows that would be outside the shadow range. This can happen when + // the user address is near the bottom, top, or shadow gap of the address + // space. + if (!AddrIsInShadow(row_shadow_addr)) continue; const char *prefix = (i == 0) ? "=>" : " "; - PrintShadowBytes(&str, prefix, (u8 *)(aligned_shadow + i * n_bytes_per_row), - (u8 *)shadow_addr, n_bytes_per_row); + PrintShadowBytes(&str, prefix, (u8 *)row_shadow_addr, (u8 *)shadow_addr, + n_bytes_per_row); } if (flags()->print_legend) PrintLegend(&str); Printf("%s", str.data()); @@ -491,13 +457,13 @@ void ErrorGeneric::Print() { uptr addr = addr_description.Address(); Report("ERROR: AddressSanitizer: %s on address %p at pc %p bp %p sp %p\n", bug_descr, (void *)addr, pc, bp, sp); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); char tname[128]; Printf("%s%s of size %zu at %p thread T%d%s%s\n", d.Access(), access_size ? (is_write ? "WRITE" : "READ") : "ACCESS", access_size, (void *)addr, tid, - ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.EndAccess()); + ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.Default()); scariness.Print(); GET_STACK_TRACE_FATAL(pc, bp); diff --git a/contrib/compiler-rt/lib/asan/asan_errors.h b/contrib/compiler-rt/lib/asan/asan_errors.h index 9a124924470e..518ba0c6945e 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.h +++ b/contrib/compiler-rt/lib/asan/asan_errors.h @@ -27,61 +27,28 @@ struct ErrorBase { u32 tid; }; -struct ErrorStackOverflow : ErrorBase { - uptr addr, pc, bp, sp; - // ErrorStackOverflow never owns the context. - void *context; - // VS2013 doesn't implement unrestricted unions, so we need a trivial default - // constructor - ErrorStackOverflow() = default; - ErrorStackOverflow(u32 tid, const SignalContext &sig) - : ErrorBase(tid), - addr(sig.addr), - pc(sig.pc), - bp(sig.bp), - sp(sig.sp), - context(sig.context) { - scariness.Clear(); - scariness.Scare(10, "stack-overflow"); - } - void Print(); -}; - struct ErrorDeadlySignal : ErrorBase { - uptr addr, pc, bp, sp; - // ErrorDeadlySignal never owns the context. - void *context; - int signo; - SignalContext::WriteFlag write_flag; - bool is_memory_access; + SignalContext signal; // VS2013 doesn't implement unrestricted unions, so we need a trivial default // constructor ErrorDeadlySignal() = default; - ErrorDeadlySignal(u32 tid, const SignalContext &sig, int signo_) - : ErrorBase(tid), - addr(sig.addr), - pc(sig.pc), - bp(sig.bp), - sp(sig.sp), - context(sig.context), - signo(signo_), - write_flag(sig.write_flag), - is_memory_access(sig.is_memory_access) { + ErrorDeadlySignal(u32 tid, const SignalContext &sig) + : ErrorBase(tid), signal(sig) { scariness.Clear(); - if (is_memory_access) { - if (addr < GetPageSizeCached()) { - scariness.Scare(10, "null-deref"); - } else if (addr == pc) { - scariness.Scare(60, "wild-jump"); - } else if (write_flag == SignalContext::WRITE) { - scariness.Scare(30, "wild-addr-write"); - } else if (write_flag == SignalContext::READ) { - scariness.Scare(20, "wild-addr-read"); - } else { - scariness.Scare(25, "wild-addr"); - } - } else { + if (signal.IsStackOverflow()) { + scariness.Scare(10, "stack-overflow"); + } else if (!signal.is_memory_access) { scariness.Scare(10, "signal"); + } else if (signal.addr < GetPageSizeCached()) { + scariness.Scare(10, "null-deref"); + } else if (signal.addr == signal.pc) { + scariness.Scare(60, "wild-jump"); + } else if (signal.write_flag == SignalContext::WRITE) { + scariness.Scare(30, "wild-addr-write"); + } else if (signal.write_flag == SignalContext::READ) { + scariness.Scare(20, "wild-addr-read"); + } else { + scariness.Scare(25, "wild-addr"); } } void Print(); @@ -104,17 +71,19 @@ struct ErrorDoubleFree : ErrorBase { void Print(); }; -struct ErrorNewDeleteSizeMismatch : ErrorBase { - // ErrorNewDeleteSizeMismatch doesn't own the stack trace. +struct ErrorNewDeleteTypeMismatch : ErrorBase { + // ErrorNewDeleteTypeMismatch doesn't own the stack trace. const BufferedStackTrace *free_stack; HeapAddressDescription addr_description; uptr delete_size; + uptr delete_alignment; // VS2013 doesn't implement unrestricted unions, so we need a trivial default // constructor - ErrorNewDeleteSizeMismatch() = default; - ErrorNewDeleteSizeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr, - uptr delete_size_) - : ErrorBase(tid), free_stack(stack), delete_size(delete_size_) { + ErrorNewDeleteTypeMismatch() = default; + ErrorNewDeleteTypeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr, + uptr delete_size_, uptr delete_alignment_) + : ErrorBase(tid), free_stack(stack), delete_size(delete_size_), + delete_alignment(delete_alignment_) { GetHeapAddressInformation(addr, 1, &addr_description); scariness.Clear(); scariness.Scare(10, "new-delete-type-mismatch"); @@ -324,10 +293,9 @@ struct ErrorGeneric : ErrorBase { // clang-format off #define ASAN_FOR_EACH_ERROR_KIND(macro) \ - macro(StackOverflow) \ macro(DeadlySignal) \ macro(DoubleFree) \ - macro(NewDeleteSizeMismatch) \ + macro(NewDeleteTypeMismatch) \ macro(FreeNotMalloced) \ macro(AllocTypeMismatch) \ macro(MallocUsableSizeNotOwned) \ diff --git a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc index 017b7d2af129..1c6184e3c7fc 100644 --- a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc +++ b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc @@ -28,9 +28,9 @@ static const u64 kAllocaRedzoneMask = 31UL; // For small size classes inline PoisonShadow for better performance. ALWAYS_INLINE void SetShadow(uptr ptr, uptr size, uptr class_id, u64 magic) { - CHECK_EQ(SHADOW_SCALE, 3); // This code expects SHADOW_SCALE=3. u64 *shadow = reinterpret_cast<u64*>(MemToShadow(ptr)); - if (class_id <= 6) { + if (SHADOW_SCALE == 3 && class_id <= 6) { + // This code expects SHADOW_SCALE=3. for (uptr i = 0; i < (((uptr)1) << class_id); i++) { shadow[i] = magic; // Make sure this does not become memset. @@ -171,7 +171,7 @@ void FakeStack::ForEachFakeFrame(RangeIteratorCallback callback, void *arg) { } } -#if SANITIZER_LINUX && !SANITIZER_ANDROID +#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA static THREADLOCAL FakeStack *fake_stack_tls; FakeStack *GetTLSFakeStack() { @@ -183,7 +183,7 @@ void SetTLSFakeStack(FakeStack *fs) { #else FakeStack *GetTLSFakeStack() { return 0; } void SetTLSFakeStack(FakeStack *fs) { } -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA static FakeStack *GetFakeStack() { AsanThread *t = GetCurrentThread(); diff --git a/contrib/compiler-rt/lib/asan/asan_flags.cc b/contrib/compiler-rt/lib/asan/asan_flags.cc index 6be0d6e94b9a..562168e6d428 100644 --- a/contrib/compiler-rt/lib/asan/asan_flags.cc +++ b/contrib/compiler-rt/lib/asan/asan_flags.cc @@ -118,6 +118,10 @@ void InitializeFlags() { const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); ubsan_parser.ParseString(ubsan_default_options); #endif +#if CAN_SANITIZE_LEAKS + const char *lsan_default_options = __lsan::MaybeCallLsanDefaultOptions(); + lsan_parser.ParseString(lsan_default_options); +#endif // Override from command line. asan_parser.ParseString(GetEnv("ASAN_OPTIONS")); @@ -144,6 +148,9 @@ void InitializeFlags() { SanitizerToolName); Die(); } + // Ensure that redzone is at least SHADOW_GRANULARITY. + if (f->redzone < (int)SHADOW_GRANULARITY) + f->redzone = SHADOW_GRANULARITY; // Make "strict_init_order" imply "check_initialization_order". // TODO(samsonov): Use a single runtime flag for an init-order checker. if (f->strict_init_order) { diff --git a/contrib/compiler-rt/lib/asan/asan_flags.inc b/contrib/compiler-rt/lib/asan/asan_flags.inc index f2216c2e9b3b..00071d39f041 100644 --- a/contrib/compiler-rt/lib/asan/asan_flags.inc +++ b/contrib/compiler-rt/lib/asan/asan_flags.inc @@ -79,6 +79,10 @@ ASAN_FLAG( "Number of seconds to sleep between printing an error report and " "terminating the program. Useful for debugging purposes (e.g. when one " "needs to attach gdb).") +ASAN_FLAG( + int, sleep_after_init, 0, + "Number of seconds to sleep after AddressSanitizer is initialized. " + "Useful for debugging purposes (e.g. when one needs to attach gdb).") ASAN_FLAG(bool, check_malloc_usable_size, true, "Allows the users to work around the bug in Nvidia drivers prior to " "295.*.") @@ -143,11 +147,6 @@ ASAN_FLAG(int, detect_odr_violation, 2, "If >=2, detect violation of One-Definition-Rule (ODR); " "If ==1, detect ODR-violation only if the two variables " "have different sizes") -ASAN_FLAG(bool, dump_instruction_bytes, false, - "If true, dump 16 bytes starting at the instruction that caused SEGV") -ASAN_FLAG(bool, dump_registers, true, - "If true, dump values of CPU registers when SEGV happens. Only " - "available on OS X for now.") ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.") ASAN_FLAG(bool, halt_on_error, true, "Crash the program after printing the first error report " diff --git a/contrib/compiler-rt/lib/asan/asan_fuchsia.cc b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc new file mode 100644 index 000000000000..0b5bff4f565e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc @@ -0,0 +1,218 @@ +//===-- asan_fuchsia.cc --------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Fuchsia-specific details. +//===---------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_fuchsia.h" +#if SANITIZER_FUCHSIA + +#include "asan_interceptors.h" +#include "asan_internal.h" +#include "asan_stack.h" +#include "asan_thread.h" + +#include <limits.h> +#include <zircon/sanitizer.h> +#include <zircon/syscalls.h> +#include <zircon/threads.h> + +namespace __asan { + +// The system already set up the shadow memory for us. +// __sanitizer::GetMaxUserVirtualAddress has already been called by +// AsanInitInternal->InitializeHighMemEnd (asan_rtl.cc). +// Just do some additional sanity checks here. +void InitializeShadowMemory() { + if (Verbosity()) PrintAddressSpaceLayout(); + + // Make sure SHADOW_OFFSET doesn't use __asan_shadow_memory_dynamic_address. + __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel; + DCHECK(kLowShadowBeg != kDefaultShadowSentinel); + __asan_shadow_memory_dynamic_address = kLowShadowBeg; + + CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); + CHECK_EQ(kHighMemEnd, __sanitizer::ShadowBounds.memory_limit - 1); + CHECK_EQ(kHighMemBeg, __sanitizer::ShadowBounds.shadow_limit); + CHECK_EQ(kHighShadowBeg, __sanitizer::ShadowBounds.shadow_base); + CHECK_EQ(kShadowGapEnd, __sanitizer::ShadowBounds.shadow_base - 1); + CHECK_EQ(kLowShadowEnd, 0); + CHECK_EQ(kLowShadowBeg, 0); +} + +void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { + UNIMPLEMENTED(); +} + +void AsanCheckDynamicRTPrereqs() {} +void AsanCheckIncompatibleRT() {} +void InitializeAsanInterceptors() {} + +void *AsanDoesNotSupportStaticLinkage() { return nullptr; } + +void InitializePlatformExceptionHandlers() {} +void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { + UNIMPLEMENTED(); +} + +// We can use a plain thread_local variable for TSD. +static thread_local void *per_thread; + +void *AsanTSDGet() { return per_thread; } + +void AsanTSDSet(void *tsd) { per_thread = tsd; } + +// There's no initialization needed, and the passed-in destructor +// will never be called. Instead, our own thread destruction hook +// (below) will call AsanThread::TSDDtor directly. +void AsanTSDInit(void (*destructor)(void *tsd)) { + DCHECK(destructor == &PlatformTSDDtor); +} + +void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); } + +static inline size_t AsanThreadMmapSize() { + return RoundUpTo(sizeof(AsanThread), PAGE_SIZE); +} + +struct AsanThread::InitOptions { + uptr stack_bottom, stack_size; +}; + +// Shared setup between thread creation and startup for the initial thread. +static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid, + uptr user_id, bool detached, + const char *name, uptr stack_bottom, + uptr stack_size) { + // In lieu of AsanThread::Create. + AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__); + + AsanThreadContext::CreateThreadContextArgs args = {thread, stack}; + u32 tid = + asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args); + asanThreadRegistry().SetThreadName(tid, name); + + // On other systems, AsanThread::Init() is called from the new + // thread itself. But on Fuchsia we already know the stack address + // range beforehand, so we can do most of the setup right now. + const AsanThread::InitOptions options = {stack_bottom, stack_size}; + thread->Init(&options); + + return thread; +} + +// This gets the same arguments passed to Init by CreateAsanThread, above. +// We're in the creator thread before the new thread is actually started, +// but its stack address range is already known. We don't bother tracking +// the static TLS address range because the system itself already uses an +// ASan-aware allocator for that. +void AsanThread::SetThreadStackAndTls(const AsanThread::InitOptions *options) { + DCHECK_NE(GetCurrentThread(), this); + DCHECK_NE(GetCurrentThread(), nullptr); + CHECK_NE(options->stack_bottom, 0); + CHECK_NE(options->stack_size, 0); + stack_bottom_ = options->stack_bottom; + stack_top_ = options->stack_bottom + options->stack_size; +} + +// Called by __asan::AsanInitInternal (asan_rtl.c). +AsanThread *CreateMainThread() { + thrd_t self = thrd_current(); + char name[ZX_MAX_NAME_LEN]; + CHECK_NE(__sanitizer::MainThreadStackBase, 0); + CHECK_GT(__sanitizer::MainThreadStackSize, 0); + AsanThread *t = CreateAsanThread( + nullptr, 0, reinterpret_cast<uptr>(self), true, + _zx_object_get_property(thrd_get_zx_handle(self), ZX_PROP_NAME, name, + sizeof(name)) == ZX_OK + ? name + : nullptr, + __sanitizer::MainThreadStackBase, __sanitizer::MainThreadStackSize); + SetCurrentThread(t); + return t; +} + +// This is called before each thread creation is attempted. So, in +// its first call, the calling thread is the initial and sole thread. +static void *BeforeThreadCreateHook(uptr user_id, bool detached, + const char *name, uptr stack_bottom, + uptr stack_size) { + EnsureMainThreadIDIsCorrect(); + // Strict init-order checking is thread-hostile. + if (flags()->strict_init_order) StopInitOrderChecking(); + + GET_STACK_TRACE_THREAD; + u32 parent_tid = GetCurrentTidOrInvalid(); + + return CreateAsanThread(&stack, parent_tid, user_id, detached, name, + stack_bottom, stack_size); +} + +// This is called after creating a new thread (in the creating thread), +// with the pointer returned by BeforeThreadCreateHook (above). +static void ThreadCreateHook(void *hook, bool aborted) { + AsanThread *thread = static_cast<AsanThread *>(hook); + if (!aborted) { + // The thread was created successfully. + // ThreadStartHook is already running in the new thread. + } else { + // The thread wasn't created after all. + // Clean up everything we set up in BeforeThreadCreateHook. + asanThreadRegistry().FinishThread(thread->tid()); + UnmapOrDie(thread, AsanThreadMmapSize()); + } +} + +// This is called in the newly-created thread before it runs anything else, +// with the pointer returned by BeforeThreadCreateHook (above). +// cf. asan_interceptors.cc:asan_thread_start +static void ThreadStartHook(void *hook, uptr os_id) { + AsanThread *thread = static_cast<AsanThread *>(hook); + SetCurrentThread(thread); + + // In lieu of AsanThread::ThreadStart. + asanThreadRegistry().StartThread(thread->tid(), os_id, /*workerthread*/ false, + nullptr); +} + +// Each thread runs this just before it exits, +// with the pointer returned by BeforeThreadCreateHook (above). +// All per-thread destructors have already been called. +static void ThreadExitHook(void *hook, uptr os_id) { + AsanThread::TSDDtor(per_thread); +} + +} // namespace __asan + +// These are declared (in extern "C") by <zircon/sanitizer.h>. +// The system runtime will call our definitions directly. + +void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached, + const char *name, void *stack_base, + size_t stack_size) { + return __asan::BeforeThreadCreateHook( + reinterpret_cast<uptr>(thread), detached, name, + reinterpret_cast<uptr>(stack_base), stack_size); +} + +void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) { + __asan::ThreadCreateHook(hook, error != thrd_success); +} + +void __sanitizer_thread_start_hook(void *hook, thrd_t self) { + __asan::ThreadStartHook(hook, reinterpret_cast<uptr>(self)); +} + +void __sanitizer_thread_exit_hook(void *hook, thrd_t self) { + __asan::ThreadExitHook(hook, reinterpret_cast<uptr>(self)); +} + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_globals.cc b/contrib/compiler-rt/lib/asan/asan_globals.cc index eebada804f07..0db65d010349 100644 --- a/contrib/compiler-rt/lib/asan/asan_globals.cc +++ b/contrib/compiler-rt/lib/asan/asan_globals.cc @@ -384,6 +384,10 @@ void __asan_register_globals(__asan_global *globals, uptr n) { } RegisterGlobal(&globals[i]); } + + // Poison the metadata. It should not be accessible to user code. + PoisonShadow(reinterpret_cast<uptr>(globals), n * sizeof(__asan_global), + kAsanGlobalRedzoneMagic); } // Unregister an array of globals. @@ -399,6 +403,9 @@ void __asan_unregister_globals(__asan_global *globals, uptr n) { } UnregisterGlobal(&globals[i]); } + + // Unpoison the metadata. + PoisonShadow(reinterpret_cast<uptr>(globals), n * sizeof(__asan_global), 0); } // This method runs immediately prior to dynamic initialization in each TU, diff --git a/contrib/compiler-rt/lib/asan/asan_init_version.h b/contrib/compiler-rt/lib/asan/asan_init_version.h index f48cc19cc515..c49fcd740248 100644 --- a/contrib/compiler-rt/lib/asan/asan_init_version.h +++ b/contrib/compiler-rt/lib/asan/asan_init_version.h @@ -15,6 +15,8 @@ #ifndef ASAN_INIT_VERSION_H #define ASAN_INIT_VERSION_H +#include "sanitizer_common/sanitizer_platform.h" + extern "C" { // Every time the ASan ABI changes we also change the version number in the // __asan_init function name. Objects built with incompatible ASan ABI @@ -32,7 +34,12 @@ extern "C" { // v6=>v7: added 'odr_indicator' to __asan_global // v7=>v8: added '__asan_(un)register_image_globals' functions for dead // stripping support on Mach-O platforms +#if SANITIZER_WORDSIZE == 32 && SANITIZER_ANDROID + // v8=>v9: 32-bit Android switched to dynamic shadow + #define __asan_version_mismatch_check __asan_version_mismatch_check_v9 +#else #define __asan_version_mismatch_check __asan_version_mismatch_check_v8 +#endif } #endif // ASAN_INIT_VERSION_H diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors.cc b/contrib/compiler-rt/lib/asan/asan_interceptors.cc index 34ca22b8616e..cb7dcb3b1ca8 100644 --- a/contrib/compiler-rt/lib/asan/asan_interceptors.cc +++ b/contrib/compiler-rt/lib/asan/asan_interceptors.cc @@ -24,6 +24,11 @@ #include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_libc.h" +// There is no general interception at all on Fuchsia. +// Only the functions in asan_interceptors_memintrinsics.cc are +// really defined to replace libc functions. +#if !SANITIZER_FUCHSIA + #if SANITIZER_POSIX #include "sanitizer_common/sanitizer_posix.h" #endif @@ -36,108 +41,6 @@ namespace __asan { -// Return true if we can quickly decide that the region is unpoisoned. -// We assume that a redzone is at least 16 bytes. -static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) { - if (size == 0) return true; - if (size <= 32) - return !AddressIsPoisoned(beg) && - !AddressIsPoisoned(beg + size - 1) && - !AddressIsPoisoned(beg + size / 2); - if (size <= 64) - return !AddressIsPoisoned(beg) && - !AddressIsPoisoned(beg + size / 4) && - !AddressIsPoisoned(beg + size - 1) && - !AddressIsPoisoned(beg + 3 * size / 4) && - !AddressIsPoisoned(beg + size / 2); - return false; -} - -struct AsanInterceptorContext { - const char *interceptor_name; -}; - -// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE, -// and ASAN_WRITE_RANGE as macro instead of function so -// that no extra frames are created, and stack trace contains -// relevant information only. -// We check all shadow bytes. -#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \ - uptr __offset = (uptr)(offset); \ - uptr __size = (uptr)(size); \ - uptr __bad = 0; \ - if (__offset > __offset + __size) { \ - GET_STACK_TRACE_FATAL_HERE; \ - ReportStringFunctionSizeOverflow(__offset, __size, &stack); \ - } \ - if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \ - (__bad = __asan_region_is_poisoned(__offset, __size))) { \ - AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \ - bool suppressed = false; \ - if (_ctx) { \ - suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \ - if (!suppressed && HaveStackTraceBasedSuppressions()) { \ - GET_STACK_TRACE_FATAL_HERE; \ - suppressed = IsStackTraceSuppressed(&stack); \ - } \ - } \ - if (!suppressed) { \ - GET_CURRENT_PC_BP_SP; \ - ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\ - } \ - } \ - } while (0) - -// memcpy is called during __asan_init() from the internals of printf(...). -// We do not treat memcpy with to==from as a bug. -// See http://llvm.org/bugs/show_bug.cgi?id=11763. -#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \ - if (asan_init_is_running) { \ - return REAL(memcpy)(to, from, size); \ - } \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - if (to != from) { \ - CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ - } \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return REAL(memcpy)(to, from, size); \ - } while (0) - -// memset is called inside Printf. -#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \ - if (asan_init_is_running) { \ - return REAL(memset)(block, c, size); \ - } \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - ASAN_WRITE_RANGE(ctx, block, size); \ - } \ - return REAL(memset)(block, c, size); \ - } while (0) - -#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return internal_memmove(to, from, size); \ - } while (0) - -#define ASAN_READ_RANGE(ctx, offset, size) \ - ACCESS_MEMORY_RANGE(ctx, offset, size, false) -#define ASAN_WRITE_RANGE(ctx, offset, size) \ - ACCESS_MEMORY_RANGE(ctx, offset, size, true) - #define ASAN_READ_STRING_OF_LEN(ctx, s, len, n) \ ASAN_READ_RANGE((ctx), (s), \ common_flags()->strict_string_checks ? (len) + 1 : (n)) @@ -145,23 +48,6 @@ struct AsanInterceptorContext { #define ASAN_READ_STRING(ctx, s, n) \ ASAN_READ_STRING_OF_LEN((ctx), (s), REAL(strlen)(s), (n)) -// Behavior of functions like "memcpy" or "strcpy" is undefined -// if memory intervals overlap. We report error in this case. -// Macro is used to avoid creation of new frames. -static inline bool RangesOverlap(const char *offset1, uptr length1, - const char *offset2, uptr length2) { - return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1)); -} -#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \ - const char *offset1 = (const char*)_offset1; \ - const char *offset2 = (const char*)_offset2; \ - if (RangesOverlap(offset1, length1, offset2, length2)) { \ - GET_STACK_TRACE_FATAL_HERE; \ - ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \ - offset2, length2, &stack); \ - } \ -} while (0) - static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) { #if SANITIZER_INTERCEPT_STRNLEN if (REAL(strnlen)) { @@ -275,6 +161,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) } while (false) #include "sanitizer_common/sanitizer_common_interceptors.inc" +#include "sanitizer_common/sanitizer_signal_interceptors.inc" // Syscall interceptors don't have contexts, we don't support suppressions // for them. @@ -356,42 +243,6 @@ INTERCEPTOR(int, pthread_join, void *t, void **arg) { DEFINE_REAL_PTHREAD_FUNCTIONS #endif // ASAN_INTERCEPT_PTHREAD_CREATE -#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION - -#if SANITIZER_ANDROID -INTERCEPTOR(void*, bsd_signal, int signum, void *handler) { - if (GetHandleSignalMode(signum) != kHandleSignalExclusive) - return REAL(bsd_signal)(signum, handler); - return 0; -} -#endif - -INTERCEPTOR(void*, signal, int signum, void *handler) { - if (GetHandleSignalMode(signum) != kHandleSignalExclusive) - return REAL(signal)(signum, handler); - return nullptr; -} - -INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) { - if (GetHandleSignalMode(signum) != kHandleSignalExclusive) - return REAL(sigaction)(signum, act, oldact); - return 0; -} - -namespace __sanitizer { -int real_sigaction(int signum, const void *act, void *oldact) { - return REAL(sigaction)(signum, (const struct sigaction *)act, - (struct sigaction *)oldact); -} -} // namespace __sanitizer - -#elif SANITIZER_POSIX -// We need to have defined REAL(sigaction) on posix systems. -DEFINE_REAL(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) -#endif // ASAN_INTERCEPT_SIGNAL_AND_SIGACTION - #if ASAN_INTERCEPT_SWAPCONTEXT static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) { // Align to page size. @@ -428,6 +279,11 @@ INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp, } #endif // ASAN_INTERCEPT_SWAPCONTEXT +#if SANITIZER_NETBSD +#define longjmp __longjmp14 +#define siglongjmp __siglongjmp14 +#endif + INTERCEPTOR(void, longjmp, void *env, int val) { __asan_handle_no_return(); REAL(longjmp)(env, val); @@ -462,18 +318,6 @@ INTERCEPTOR(void, __cxa_throw, void *a, void *b, void *c) { } #endif -void *__asan_memcpy(void *to, const void *from, uptr size) { - ASAN_MEMCPY_IMPL(nullptr, to, from, size); -} - -void *__asan_memset(void *block, int c, uptr size) { - ASAN_MEMSET_IMPL(nullptr, block, c, size); -} - -void *__asan_memmove(void *to, const void *from, uptr size) { - ASAN_MEMMOVE_IMPL(nullptr, to, from, size); -} - #if ASAN_INTERCEPT_INDEX # if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX INTERCEPTOR(char*, index, const char *string, int c) @@ -711,6 +555,7 @@ void InitializeAsanInterceptors() { CHECK(!was_called_once); was_called_once = true; InitializeCommonInterceptors(); + InitializeSignalInterceptors(); // Intercept str* functions. ASAN_INTERCEPT_FUNC(strcat); // NOLINT @@ -733,15 +578,9 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(strtoll); #endif - // Intecept signal- and jump-related functions. + // Intecept jump-related functions. ASAN_INTERCEPT_FUNC(longjmp); -#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION - ASAN_INTERCEPT_FUNC(sigaction); -#if SANITIZER_ANDROID - ASAN_INTERCEPT_FUNC(bsd_signal); -#endif - ASAN_INTERCEPT_FUNC(signal); -#endif + #if ASAN_INTERCEPT_SWAPCONTEXT ASAN_INTERCEPT_FUNC(swapcontext); #endif @@ -785,3 +624,5 @@ void InitializeAsanInterceptors() { } } // namespace __asan + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors.h b/contrib/compiler-rt/lib/asan/asan_interceptors.h index 93fca4f67366..e13bdecfdede 100644 --- a/contrib/compiler-rt/lib/asan/asan_interceptors.h +++ b/contrib/compiler-rt/lib/asan/asan_interceptors.h @@ -15,9 +15,30 @@ #define ASAN_INTERCEPTORS_H #include "asan_internal.h" +#include "asan_interceptors_memintrinsics.h" #include "interception/interception.h" #include "sanitizer_common/sanitizer_platform_interceptors.h" +namespace __asan { + +void InitializeAsanInterceptors(); +void InitializePlatformInterceptors(); + +#define ENSURE_ASAN_INITED() \ + do { \ + CHECK(!asan_init_is_running); \ + if (UNLIKELY(!asan_inited)) { \ + AsanInitFromRtl(); \ + } \ + } while (0) + +} // namespace __asan + +// There is no general interception at all on Fuchsia. +// Only the functions in asan_interceptors_memintrinsics.h are +// really defined to replace libc functions. +#if !SANITIZER_FUCHSIA + // Use macro to describe if specific function should be // intercepted on a given platform. #if !SANITIZER_WINDOWS @@ -34,25 +55,20 @@ # define ASAN_INTERCEPT_FORK 0 #endif -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS # define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 1 #else # define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 0 #endif -#if SANITIZER_LINUX && !SANITIZER_ANDROID +#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_SOLARIS # define ASAN_INTERCEPT_SWAPCONTEXT 1 #else # define ASAN_INTERCEPT_SWAPCONTEXT 0 #endif #if !SANITIZER_WINDOWS -# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 1 -#else -# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 0 -#endif - -#if !SANITIZER_WINDOWS # define ASAN_INTERCEPT_SIGLONGJMP 1 #else # define ASAN_INTERCEPT_SIGLONGJMP 0 @@ -66,7 +82,8 @@ // Android bug: https://code.google.com/p/android/issues/detail?id=61799 #if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && \ - !(SANITIZER_ANDROID && defined(__i386)) + !(SANITIZER_ANDROID && defined(__i386)) && \ + !SANITIZER_SOLARIS # define ASAN_INTERCEPT___CXA_THROW 1 #else # define ASAN_INTERCEPT___CXA_THROW 0 @@ -85,16 +102,11 @@ #endif DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size) -DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size) -DECLARE_REAL(void*, memset, void *block, int c, uptr size) DECLARE_REAL(char*, strchr, const char *str, int c) DECLARE_REAL(SIZE_T, strlen, const char *s) DECLARE_REAL(char*, strncpy, char *to, const char *from, uptr size) DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen) DECLARE_REAL(char*, strstr, const char *s1, const char *s2) -struct sigaction; -DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) #if !SANITIZER_MAC #define ASAN_INTERCEPT_FUNC(name) \ @@ -113,18 +125,6 @@ DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, #define ASAN_INTERCEPT_FUNC(name) #endif // SANITIZER_MAC -namespace __asan { - -void InitializeAsanInterceptors(); -void InitializePlatformInterceptors(); - -#define ENSURE_ASAN_INITED() do { \ - CHECK(!asan_init_is_running); \ - if (UNLIKELY(!asan_inited)) { \ - AsanInitFromRtl(); \ - } \ -} while (0) - -} // namespace __asan +#endif // !SANITIZER_FUCHSIA #endif // ASAN_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc new file mode 100644 index 000000000000..c89cb011492e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc @@ -0,0 +1,44 @@ +//===-- asan_interceptors_memintrinsics.cc --------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// ASan versions of memcpy, memmove, and memset. +//===---------------------------------------------------------------------===// + +#include "asan_interceptors_memintrinsics.h" +#include "asan_report.h" +#include "asan_stack.h" +#include "asan_suppressions.h" + +using namespace __asan; // NOLINT + +void *__asan_memcpy(void *to, const void *from, uptr size) { + ASAN_MEMCPY_IMPL(nullptr, to, from, size); +} + +void *__asan_memset(void *block, int c, uptr size) { + ASAN_MEMSET_IMPL(nullptr, block, c, size); +} + +void *__asan_memmove(void *to, const void *from, uptr size) { + ASAN_MEMMOVE_IMPL(nullptr, to, from, size); +} + +#if SANITIZER_FUCHSIA + +// Fuchsia doesn't use sanitizer_common_interceptors.inc, but the only +// things there it wants are these three. Just define them as aliases +// here rather than repeating the contents. + +decltype(memcpy) memcpy[[gnu::alias("__asan_memcpy")]]; +decltype(memmove) memmove[[gnu::alias("__asan_memmove")]]; +decltype(memset) memset[[gnu::alias("__asan_memset")]]; + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h new file mode 100644 index 000000000000..5a8339a23e97 --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h @@ -0,0 +1,148 @@ +//===-- asan_interceptors_memintrinsics.h -----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// ASan-private header for asan_memintrin.cc +//===---------------------------------------------------------------------===// +#ifndef ASAN_MEMINTRIN_H +#define ASAN_MEMINTRIN_H + +#include "asan_interface_internal.h" +#include "asan_internal.h" +#include "asan_mapping.h" +#include "interception/interception.h" + +DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size) +DECLARE_REAL(void*, memset, void *block, int c, uptr size) + +namespace __asan { + +// Return true if we can quickly decide that the region is unpoisoned. +// We assume that a redzone is at least 16 bytes. +static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) { + if (size == 0) return true; + if (size <= 32) + return !AddressIsPoisoned(beg) && + !AddressIsPoisoned(beg + size - 1) && + !AddressIsPoisoned(beg + size / 2); + if (size <= 64) + return !AddressIsPoisoned(beg) && + !AddressIsPoisoned(beg + size / 4) && + !AddressIsPoisoned(beg + size - 1) && + !AddressIsPoisoned(beg + 3 * size / 4) && + !AddressIsPoisoned(beg + size / 2); + return false; +} + +struct AsanInterceptorContext { + const char *interceptor_name; +}; + +// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE, +// and ASAN_WRITE_RANGE as macro instead of function so +// that no extra frames are created, and stack trace contains +// relevant information only. +// We check all shadow bytes. +#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \ + uptr __offset = (uptr)(offset); \ + uptr __size = (uptr)(size); \ + uptr __bad = 0; \ + if (__offset > __offset + __size) { \ + GET_STACK_TRACE_FATAL_HERE; \ + ReportStringFunctionSizeOverflow(__offset, __size, &stack); \ + } \ + if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \ + (__bad = __asan_region_is_poisoned(__offset, __size))) { \ + AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \ + bool suppressed = false; \ + if (_ctx) { \ + suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \ + if (!suppressed && HaveStackTraceBasedSuppressions()) { \ + GET_STACK_TRACE_FATAL_HERE; \ + suppressed = IsStackTraceSuppressed(&stack); \ + } \ + } \ + if (!suppressed) { \ + GET_CURRENT_PC_BP_SP; \ + ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\ + } \ + } \ + } while (0) + +// memcpy is called during __asan_init() from the internals of printf(...). +// We do not treat memcpy with to==from as a bug. +// See http://llvm.org/bugs/show_bug.cgi?id=11763. +#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \ + if (asan_init_is_running) { \ + return REAL(memcpy)(to, from, size); \ + } \ + ENSURE_ASAN_INITED(); \ + if (flags()->replace_intrin) { \ + if (to != from) { \ + CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ + } \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } \ + return REAL(memcpy)(to, from, size); \ + } while (0) + +// memset is called inside Printf. +#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \ + if (asan_init_is_running) { \ + return REAL(memset)(block, c, size); \ + } \ + ENSURE_ASAN_INITED(); \ + if (flags()->replace_intrin) { \ + ASAN_WRITE_RANGE(ctx, block, size); \ + } \ + return REAL(memset)(block, c, size); \ + } while (0) + +#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \ + ENSURE_ASAN_INITED(); \ + if (flags()->replace_intrin) { \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } \ + return internal_memmove(to, from, size); \ + } while (0) + +#define ASAN_READ_RANGE(ctx, offset, size) \ + ACCESS_MEMORY_RANGE(ctx, offset, size, false) +#define ASAN_WRITE_RANGE(ctx, offset, size) \ + ACCESS_MEMORY_RANGE(ctx, offset, size, true) + +// Behavior of functions like "memcpy" or "strcpy" is undefined +// if memory intervals overlap. We report error in this case. +// Macro is used to avoid creation of new frames. +static inline bool RangesOverlap(const char *offset1, uptr length1, + const char *offset2, uptr length2) { + return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1)); +} +#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \ + const char *offset1 = (const char*)_offset1; \ + const char *offset2 = (const char*)_offset2; \ + if (RangesOverlap(offset1, length1, offset2, length2)) { \ + GET_STACK_TRACE_FATAL_HERE; \ + ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \ + offset2, length2, &stack); \ + } \ +} while (0) + +} // namespace __asan + +#endif // ASAN_MEMINTRIN_H diff --git a/contrib/compiler-rt/lib/asan/asan_internal.h b/contrib/compiler-rt/lib/asan/asan_internal.h index f09bbd83af25..19133e5291a9 100644 --- a/contrib/compiler-rt/lib/asan/asan_internal.h +++ b/contrib/compiler-rt/lib/asan/asan_internal.h @@ -69,8 +69,12 @@ void InitializePlatformExceptionHandlers(); bool IsSystemHeapAddress(uptr addr); // asan_rtl.cc +void PrintAddressSpaceLayout(); void NORETURN ShowStatsAndAbort(); +// asan_shadow_setup.cc +void InitializeShadowMemory(); + // asan_malloc_linux.cc / asan_malloc_mac.cc void ReplaceSystemMalloc(); @@ -80,6 +84,9 @@ void *AsanDoesNotSupportStaticLinkage(); void AsanCheckDynamicRTPrereqs(); void AsanCheckIncompatibleRT(); +// asan_thread.cc +AsanThread *CreateMainThread(); + // Support function for __asan_(un)register_image_globals. Searches for the // loaded image containing `needle' and then enumerates all global metadata // structures declared in that image, applying `op' (e.g., diff --git a/contrib/compiler-rt/lib/asan/asan_linux.cc b/contrib/compiler-rt/lib/asan/asan_linux.cc index 6d47ba432a61..047e1dbb72fa 100644 --- a/contrib/compiler-rt/lib/asan/asan_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_linux.cc @@ -13,10 +13,12 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "asan_interceptors.h" #include "asan_internal.h" +#include "asan_premap_shadow.h" #include "asan_thread.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_freebsd.h" @@ -39,9 +41,17 @@ #include <sys/link_elf.h> #endif -#if SANITIZER_ANDROID || SANITIZER_FREEBSD +#if SANITIZER_SOLARIS +#include <link.h> +#endif + +#if SANITIZER_ANDROID || SANITIZER_FREEBSD || SANITIZER_SOLARIS #include <ucontext.h> extern "C" void* _DYNAMIC; +#elif SANITIZER_NETBSD +#include <link_elf.h> +#include <ucontext.h> +extern Elf_Dyn _DYNAMIC; #else #include <sys/ucontext.h> #include <link.h> @@ -77,9 +87,51 @@ void *AsanDoesNotSupportStaticLinkage() { return &_DYNAMIC; // defined in link.h } +static void UnmapFromTo(uptr from, uptr to) { + CHECK(to >= from); + if (to == from) return; + uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from); + if (UNLIKELY(internal_iserror(res))) { + Report( + "ERROR: AddresSanitizer failed to unmap 0x%zx (%zd) bytes at address " + "%p\n", + to - from, to - from, from); + CHECK("unable to unmap" && 0); + } +} + +#if ASAN_PREMAP_SHADOW +uptr FindPremappedShadowStart() { + uptr granularity = GetMmapGranularity(); + uptr shadow_start = reinterpret_cast<uptr>(&__asan_shadow); + uptr premap_shadow_size = PremapShadowSize(); + uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity); + // We may have mapped too much. Release extra memory. + UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size); + return shadow_start; +} +#endif + uptr FindDynamicShadowStart() { - UNREACHABLE("FindDynamicShadowStart is not available"); - return 0; +#if ASAN_PREMAP_SHADOW + if (!PremapShadowFailed()) + return FindPremappedShadowStart(); +#endif + + uptr granularity = GetMmapGranularity(); + uptr alignment = granularity * 8; + uptr left_padding = granularity; + uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity); + uptr map_size = shadow_size + left_padding + alignment; + + uptr map_start = (uptr)MmapNoAccess(map_size); + CHECK_NE(map_start, ~(uptr)0); + + uptr shadow_start = RoundUpTo(map_start + left_padding, alignment); + UnmapFromTo(map_start, shadow_start - left_padding); + UnmapFromTo(shadow_start + shadow_size, map_start + map_size); + + return shadow_start; } void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { @@ -93,6 +145,9 @@ void AsanCheckIncompatibleRT() {} #else static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size, void *data) { + VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n", + info->dlpi_name, info->dlpi_addr); + // Continue until the first dynamic library is found if (!info->dlpi_name || info->dlpi_name[0] == 0) return 0; @@ -101,6 +156,21 @@ static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size, if (internal_strncmp(info->dlpi_name, "linux-", sizeof("linux-") - 1) == 0) return 0; +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + // Ignore first entry (the main program) + char **p = (char **)data; + if (!(*p)) { + *p = (char *)-1; + return 0; + } +#endif + +#if SANITIZER_SOLARIS + // Ignore executable on Solaris + if (info->dlpi_addr == 0) + return 0; +#endif + *(const char **)data = info->dlpi_name; return 1; } @@ -179,4 +249,5 @@ void *AsanDlSymNext(const char *sym) { } // namespace __asan -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc index fd40f47db1c4..6697ff8764ed 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc @@ -15,7 +15,8 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \ + SANITIZER_NETBSD || SANITIZER_SOLARIS #include "sanitizer_common/sanitizer_tls_get_addr.h" #include "asan_allocator.h" @@ -30,9 +31,9 @@ static uptr allocated_for_dlsym; static const uptr kDlsymAllocPoolSize = 1024; static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; -static bool IsInDlsymAllocPool(const void *ptr) { +static INLINE bool IsInDlsymAllocPool(const void *ptr) { uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); + return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); } static void *AllocateFromLocalPool(uptr size_in_bytes) { @@ -43,6 +44,26 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { return mem; } +static INLINE bool MaybeInDlsym() { + // Fuchsia doesn't use dlsym-based interceptors. + return !SANITIZER_FUCHSIA && asan_init_is_running; +} + +static void *ReallocFromLocalPool(void *ptr, uptr size) { + const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); + void *new_ptr; + if (UNLIKELY(MaybeInDlsym())) { + new_ptr = AllocateFromLocalPool(size); + } else { + ENSURE_ASAN_INITED(); + GET_STACK_TRACE_MALLOC; + new_ptr = asan_malloc(size, &stack); + } + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; +} + INTERCEPTOR(void, free, void *ptr) { GET_STACK_TRACE_FREE; if (UNLIKELY(IsInDlsymAllocPool(ptr))) @@ -60,7 +81,7 @@ INTERCEPTOR(void, cfree, void *ptr) { #endif // SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(asan_init_is_running)) + if (UNLIKELY(MaybeInDlsym())) // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. return AllocateFromLocalPool(size); ENSURE_ASAN_INITED(); @@ -69,7 +90,7 @@ INTERCEPTOR(void*, malloc, uptr size) { } INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { - if (UNLIKELY(asan_init_is_running)) + if (UNLIKELY(MaybeInDlsym())) // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. return AllocateFromLocalPool(nmemb * size); ENSURE_ASAN_INITED(); @@ -78,21 +99,9 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { } INTERCEPTOR(void*, realloc, void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(asan_init_is_running)) { - new_ptr = AllocateFromLocalPool(size); - } else { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - new_ptr = asan_malloc(size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } - if (UNLIKELY(asan_init_is_running)) + if (UNLIKELY(IsInDlsymAllocPool(ptr))) + return ReallocFromLocalPool(ptr, size); + if (UNLIKELY(MaybeInDlsym())) return AllocateFromLocalPool(size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; @@ -226,4 +235,5 @@ void ReplaceSystemMalloc() { } // namespace __asan #endif // SANITIZER_ANDROID -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || + // SANITIZER_NETBSD || SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/asan/asan_mapping.h b/contrib/compiler-rt/lib/asan/asan_mapping.h index 695740cd982f..d3f360ffc969 100644 --- a/contrib/compiler-rt/lib/asan/asan_mapping.h +++ b/contrib/compiler-rt/lib/asan/asan_mapping.h @@ -115,6 +115,13 @@ // || `[0x40000000, 0x47ffffff]` || LowShadow || // || `[0x00000000, 0x3fffffff]` || LowMem || // +// Shadow mapping on NetBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: +// || `[0x4feffffffe01, 0x7f7ffffff000]` || HighMem || +// || `[0x49fdffffffc0, 0x4feffffffe00]` || HighShadow || +// || `[0x480000000000, 0x49fdffffffbf]` || ShadowGap || +// || `[0x400000000000, 0x47ffffffffff]` || LowShadow || +// || `[0x000000000000, 0x3fffffffffff]` || LowMem || +// // Default Windows/i386 mapping: // (the exact location of HighShadow/HighMem may vary depending // on WoW64, /LARGEADDRESSAWARE, etc). @@ -124,11 +131,16 @@ // || `[0x30000000, 0x35ffffff]` || LowShadow || // || `[0x00000000, 0x2fffffff]` || LowMem || +#if defined(ASAN_SHADOW_SCALE) +static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE; +#else static const u64 kDefaultShadowScale = 3; +#endif static const u64 kDefaultShadowSentinel = ~(uptr)0; static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000 static const u64 kDefaultShadowOffset64 = 1ULL << 44; -static const u64 kDefaultShort64bitShadowOffset = 0x7FFF8000; // < 2G. +static const u64 kDefaultShort64bitShadowOffset = + 0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G. static const u64 kIosShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kIosShadowOffset64 = 0x120200000; static const u64 kIosSimShadowOffset32 = 1ULL << 30; @@ -136,18 +148,20 @@ static const u64 kIosSimShadowOffset64 = kDefaultShadowOffset64; static const u64 kAArch64_ShadowOffset64 = 1ULL << 36; static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000; static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37; -static const u64 kPPC64_ShadowOffset64 = 1ULL << 41; +static const u64 kPPC64_ShadowOffset64 = 1ULL << 44; static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52; static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 +static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 #define SHADOW_SCALE kDefaultShadowScale - -#if SANITIZER_WORDSIZE == 32 +#if SANITIZER_FUCHSIA +# define SHADOW_OFFSET (0) +#elif SANITIZER_WORDSIZE == 32 # if SANITIZER_ANDROID -# define SHADOW_OFFSET (0) +# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address # elif defined(__mips__) # define SHADOW_OFFSET kMIPS32_ShadowOffset32 # elif SANITIZER_FREEBSD @@ -178,6 +192,8 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 # define SHADOW_OFFSET kSystemZ_ShadowOffset64 # elif SANITIZER_FREEBSD # define SHADOW_OFFSET kFreeBSD_ShadowOffset64 +# elif SANITIZER_NETBSD +# define SHADOW_OFFSET kNetBSD_ShadowOffset64 # elif SANITIZER_MAC # define SHADOW_OFFSET kDefaultShadowOffset64 # elif defined(__mips64) @@ -189,6 +205,12 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 # endif #endif +#if SANITIZER_ANDROID && defined(__arm__) +# define ASAN_PREMAP_SHADOW 1 +#else +# define ASAN_PREMAP_SHADOW 0 +#endif + #define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE) #define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET)) diff --git a/contrib/compiler-rt/lib/asan/asan_memory_profile.cc b/contrib/compiler-rt/lib/asan/asan_memory_profile.cc index 05846c37cb6d..603284c8c688 100644 --- a/contrib/compiler-rt/lib/asan/asan_memory_profile.cc +++ b/contrib/compiler-rt/lib/asan/asan_memory_profile.cc @@ -107,6 +107,9 @@ static void MemoryProfileCB(const SuspendedThreadsList &suspended_threads_list, __lsan::ForEachChunk(ChunkCallback, &hp); uptr *Arg = reinterpret_cast<uptr*>(argument); hp.Print(Arg[0], Arg[1]); + + if (Verbosity()) + __asan_print_accumulated_stats(); } } // namespace __asan diff --git a/contrib/compiler-rt/lib/asan/asan_new_delete.cc b/contrib/compiler-rt/lib/asan/asan_new_delete.cc index e68c7f3e2400..072f027addd6 100644 --- a/contrib/compiler-rt/lib/asan/asan_new_delete.cc +++ b/contrib/compiler-rt/lib/asan/asan_new_delete.cc @@ -125,77 +125,69 @@ INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) { INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); } -#endif +#endif // !SANITIZER_MAC #define OPERATOR_DELETE_BODY(type) \ GET_STACK_TRACE_FREE;\ - asan_free(ptr, &stack, type); + asan_delete(ptr, 0, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE(type) \ + GET_STACK_TRACE_FREE;\ + asan_delete(ptr, size, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_ALIGN(type) \ + GET_STACK_TRACE_FREE;\ + asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \ + GET_STACK_TRACE_FREE;\ + asan_delete(ptr, size, static_cast<uptr>(align), &stack, type); #if !SANITIZER_MAC CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr) NOEXCEPT +{ OPERATOR_DELETE_BODY(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr) NOEXCEPT +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, size_t size) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW); -} +void operator delete(void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, size_t size) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW_BR); -} +void operator delete[](void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::align_val_t) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::align_val_t) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::align_val_t, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::align_val_t, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, size_t size, std::align_val_t) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW); -} +void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW_BR); -} +void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); } #else // SANITIZER_MAC -INTERCEPTOR(void, _ZdlPv, void *ptr) { - OPERATOR_DELETE_BODY(FROM_NEW); -} -INTERCEPTOR(void, _ZdaPv, void *ptr) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} -INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW); -} -INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} -#endif +INTERCEPTOR(void, _ZdlPv, void *ptr) +{ OPERATOR_DELETE_BODY(FROM_NEW); } +INTERCEPTOR(void, _ZdaPv, void *ptr) +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } +INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW); } +INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } +#endif // !SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.cc b/contrib/compiler-rt/lib/asan/asan_poisoning.cc index abb75ab3bf92..c3a82aa0049e 100644 --- a/contrib/compiler-rt/lib/asan/asan_poisoning.cc +++ b/contrib/compiler-rt/lib/asan/asan_poisoning.cc @@ -217,7 +217,7 @@ uptr __asan_region_is_poisoned(uptr beg, uptr size) { uptr __bad = __asan_region_is_poisoned(__p, __size); \ __asan_report_error(pc, bp, sp, __bad, isWrite, __size, 0);\ } \ - } while (false); \ + } while (false) extern "C" SANITIZER_INTERFACE_ATTRIBUTE diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.h b/contrib/compiler-rt/lib/asan/asan_poisoning.h index cc3281e08a71..1e00070bcf63 100644 --- a/contrib/compiler-rt/lib/asan/asan_poisoning.h +++ b/contrib/compiler-rt/lib/asan/asan_poisoning.h @@ -46,8 +46,11 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, // for mapping shadow and zeroing out pages doesn't "just work", so we should // probably provide higher-level interface for these operations. // For now, just memset on Windows. - if (value || - SANITIZER_WINDOWS == 1 || + if (value || SANITIZER_WINDOWS == 1 || + // TODO(mcgrathr): Fuchsia doesn't allow the shadow mapping to be + // changed at all. It doesn't currently have an efficient means + // to zero a bunch of pages, but maybe we should add one. + SANITIZER_FUCHSIA == 1 || shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg); } else { diff --git a/contrib/compiler-rt/lib/asan/asan_posix.cc b/contrib/compiler-rt/lib/asan/asan_posix.cc index added746ace8..17c28b0aea2a 100644 --- a/contrib/compiler-rt/lib/asan/asan_posix.cc +++ b/contrib/compiler-rt/lib/asan/asan_posix.cc @@ -25,7 +25,6 @@ #include "sanitizer_common/sanitizer_procmaps.h" #include <pthread.h> -#include <signal.h> #include <stdlib.h> #include <sys/time.h> #include <sys/resource.h> @@ -34,58 +33,9 @@ namespace __asan { void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { - ScopedDeadlySignal signal_scope(GetCurrentThread()); - int code = (int)((siginfo_t*)siginfo)->si_code; - // Write the first message using fd=2, just in case. - // It may actually fail to write in case stderr is closed. - internal_write(2, "ASAN:DEADLYSIGNAL\n", 18); - SignalContext sig = SignalContext::Create(siginfo, context); - - // Access at a reasonable offset above SP, or slightly below it (to account - // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is - // probably a stack overflow. -#ifdef __s390__ - // On s390, the fault address in siginfo points to start of the page, not - // to the precise word that was accessed. Mask off the low bits of sp to - // take it into account. - bool IsStackAccess = sig.addr >= (sig.sp & ~0xFFF) && - sig.addr < sig.sp + 0xFFFF; -#else - bool IsStackAccess = sig.addr + 512 > sig.sp && sig.addr < sig.sp + 0xFFFF; -#endif - -#if __powerpc__ - // Large stack frames can be allocated with e.g. - // lis r0,-10000 - // stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000 - // If the store faults then sp will not have been updated, so test above - // will not work, because the fault address will be more than just "slightly" - // below sp. - if (!IsStackAccess && IsAccessibleMemoryRange(sig.pc, 4)) { - u32 inst = *(unsigned *)sig.pc; - u32 ra = (inst >> 16) & 0x1F; - u32 opcd = inst >> 26; - u32 xo = (inst >> 1) & 0x3FF; - // Check for store-with-update to sp. The instructions we accept are: - // stbu rs,d(ra) stbux rs,ra,rb - // sthu rs,d(ra) sthux rs,ra,rb - // stwu rs,d(ra) stwux rs,ra,rb - // stdu rs,ds(ra) stdux rs,ra,rb - // where ra is r1 (the stack pointer). - if (ra == 1 && - (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 || - (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181)))) - IsStackAccess = true; - } -#endif // __powerpc__ - - // We also check si_code to filter out SEGV caused by something else other - // then hitting the guard page or unmapped memory, like, for example, - // unaligned memory access. - if (IsStackAccess && (code == si_SEGV_MAPERR || code == si_SEGV_ACCERR)) - ReportStackOverflow(sig); - else - ReportDeadlySignal(signo, sig); + StartReportDeadlySignal(); + SignalContext sig(siginfo, context); + ReportDeadlySignal(sig); } // ---------------------- TSD ---------------- {{{1 diff --git a/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc b/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc new file mode 100644 index 000000000000..229eba99fe0e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc @@ -0,0 +1,79 @@ +//===-- asan_premap_shadow.cc ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Reserve shadow memory with an ifunc resolver. +//===----------------------------------------------------------------------===// + +#include "asan_mapping.h" + +#if ASAN_PREMAP_SHADOW + +#include "asan_premap_shadow.h" +#include "sanitizer_common/sanitizer_posix.h" + +namespace __asan { + +// The code in this file needs to run in an unrelocated binary. It may not +// access any external symbol, including its own non-hidden globals. + +// Conservative upper limit. +uptr PremapShadowSize() { + uptr granularity = GetMmapGranularity(); + return RoundUpTo(GetMaxVirtualAddress() >> SHADOW_SCALE, granularity); +} + +// Returns an address aligned to 8 pages, such that one page on the left and +// PremapShadowSize() bytes on the right of it are mapped r/o. +uptr PremapShadow() { + uptr granularity = GetMmapGranularity(); + uptr alignment = granularity * 8; + uptr left_padding = granularity; + uptr shadow_size = PremapShadowSize(); + uptr map_size = shadow_size + left_padding + alignment; + + uptr map_start = (uptr)MmapNoAccess(map_size); + CHECK_NE(map_start, ~(uptr)0); + + uptr shadow_start = RoundUpTo(map_start + left_padding, alignment); + uptr shadow_end = shadow_start + shadow_size; + internal_munmap(reinterpret_cast<void *>(map_start), + shadow_start - left_padding - map_start); + internal_munmap(reinterpret_cast<void *>(shadow_end), + map_start + map_size - shadow_end); + return shadow_start; +} + +bool PremapShadowFailed() { + uptr shadow = reinterpret_cast<uptr>(&__asan_shadow); + uptr resolver = reinterpret_cast<uptr>(&__asan_premap_shadow); + // shadow == resolver is how Android KitKat and older handles ifunc. + // shadow == 0 just in case. + if (shadow == 0 || shadow == resolver) + return true; + return false; +} +} // namespace __asan + +extern "C" { +decltype(__asan_shadow)* __asan_premap_shadow() { + // The resolver may be called multiple times. Map the shadow just once. + static uptr premapped_shadow = 0; + if (!premapped_shadow) premapped_shadow = __asan::PremapShadow(); + return reinterpret_cast<decltype(__asan_shadow)*>(premapped_shadow); +} + +// __asan_shadow is a "function" that has the same address as the first byte of +// the shadow mapping. +INTERFACE_ATTRIBUTE __attribute__((ifunc("__asan_premap_shadow"))) void +__asan_shadow(); +} + +#endif // ASAN_PREMAP_SHADOW diff --git a/contrib/compiler-rt/lib/asan/asan_premap_shadow.h b/contrib/compiler-rt/lib/asan/asan_premap_shadow.h new file mode 100644 index 000000000000..41acbdbbb69f --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_premap_shadow.h @@ -0,0 +1,30 @@ +//===-- asan_mapping.h ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Premap shadow range with an ifunc resolver. +//===----------------------------------------------------------------------===// + + +#ifndef ASAN_PREMAP_SHADOW_H +#define ASAN_PREMAP_SHADOW_H + +#if ASAN_PREMAP_SHADOW +namespace __asan { +// Conservative upper limit. +uptr PremapShadowSize(); +bool PremapShadowFailed(); +} +#endif + +extern "C" INTERFACE_ATTRIBUTE void __asan_shadow(); +extern "C" decltype(__asan_shadow)* __asan_premap_shadow(); + +#endif // ASAN_PREMAP_SHADOW_H diff --git a/contrib/compiler-rt/lib/asan/asan_report.cc b/contrib/compiler-rt/lib/asan/asan_report.cc index 2e477f258b8d..e3bc02994efd 100644 --- a/contrib/compiler-rt/lib/asan/asan_report.cc +++ b/contrib/compiler-rt/lib/asan/asan_report.cc @@ -60,9 +60,8 @@ void PrintMemoryByte(InternalScopedString *str, const char *before, u8 byte, bool in_shadow, const char *after) { Decorator d; str->append("%s%s%x%x%s%s", before, - in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), - byte >> 4, byte & 15, - in_shadow ? d.EndShadowByte() : d.EndMemoryByte(), after); + in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), byte >> 4, + byte & 15, d.Default(), after); } static void PrintZoneForPointer(uptr ptr, uptr zone_ptr, @@ -123,53 +122,15 @@ bool ParseFrameDescription(const char *frame_descr, // immediately after printing error report. class ScopedInErrorReport { public: - explicit ScopedInErrorReport(bool fatal = false) { - halt_on_error_ = fatal || flags()->halt_on_error; - - if (lock_.TryLock()) { - StartReporting(); - return; - } - - // ASan found two bugs in different threads simultaneously. - - u32 current_tid = GetCurrentTidOrInvalid(); - if (reporting_thread_tid_ == current_tid || - reporting_thread_tid_ == kInvalidTid) { - // This is either asynch signal or nested error during error reporting. - // Fail simple to avoid deadlocks in Report(). - - // Can't use Report() here because of potential deadlocks - // in nested signal handlers. - const char msg[] = "AddressSanitizer: nested bug in the same thread, " - "aborting.\n"; - WriteToFile(kStderrFd, msg, sizeof(msg)); - - internal__exit(common_flags()->exitcode); - } - - if (halt_on_error_) { - // Do not print more than one report, otherwise they will mix up. - // Error reporting functions shouldn't return at this situation, as - // they are effectively no-returns. - - Report("AddressSanitizer: while reporting a bug found another one. " - "Ignoring.\n"); - - // Sleep long enough to make sure that the thread which started - // to print an error report will finish doing it. - SleepForSeconds(Max(100, flags()->sleep_before_dying + 1)); - - // If we're still not dead for some reason, use raw _exit() instead of - // Die() to bypass any additional checks. - internal__exit(common_flags()->exitcode); - } else { - // The other thread will eventually finish reporting - // so it's safe to wait - lock_.Lock(); - } - - StartReporting(); + explicit ScopedInErrorReport(bool fatal = false) + : halt_on_error_(fatal || flags()->halt_on_error) { + // Make sure the registry and sanitizer report mutexes are locked while + // we're printing an error report. + // We can lock them only here to avoid self-deadlock in case of + // recursive reports. + asanThreadRegistry().Lock(); + Printf( + "=================================================================\n"); } ~ScopedInErrorReport() { @@ -217,9 +178,6 @@ class ScopedInErrorReport { if (!halt_on_error_) internal_memset(¤t_error_, 0, sizeof(current_error_)); - CommonSanitizerReportMutex.Unlock(); - reporting_thread_tid_ = kInvalidTid; - lock_.Unlock(); if (halt_on_error_) { Report("ABORTING\n"); Die(); @@ -237,39 +195,18 @@ class ScopedInErrorReport { } private: - void StartReporting() { - // Make sure the registry and sanitizer report mutexes are locked while - // we're printing an error report. - // We can lock them only here to avoid self-deadlock in case of - // recursive reports. - asanThreadRegistry().Lock(); - CommonSanitizerReportMutex.Lock(); - reporting_thread_tid_ = GetCurrentTidOrInvalid(); - Printf("====================================================" - "=============\n"); - } - - static StaticSpinMutex lock_; - static u32 reporting_thread_tid_; + ScopedErrorReportLock error_report_lock_; // Error currently being reported. This enables the destructor to interact // with the debugger and point it to an error description. static ErrorDescription current_error_; bool halt_on_error_; }; -StaticSpinMutex ScopedInErrorReport::lock_; -u32 ScopedInErrorReport::reporting_thread_tid_ = kInvalidTid; ErrorDescription ScopedInErrorReport::current_error_; -void ReportStackOverflow(const SignalContext &sig) { +void ReportDeadlySignal(const SignalContext &sig) { ScopedInErrorReport in_report(/*fatal*/ true); - ErrorStackOverflow error(GetCurrentTidOrInvalid(), sig); - in_report.ReportError(error); -} - -void ReportDeadlySignal(int signo, const SignalContext &sig) { - ScopedInErrorReport in_report(/*fatal*/ true); - ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig, signo); + ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig); in_report.ReportError(error); } @@ -279,11 +216,12 @@ void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack) { in_report.ReportError(error); } -void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size, +void ReportNewDeleteTypeMismatch(uptr addr, uptr delete_size, + uptr delete_alignment, BufferedStackTrace *free_stack) { ScopedInErrorReport in_report; - ErrorNewDeleteSizeMismatch error(GetCurrentTidOrInvalid(), free_stack, addr, - delete_size); + ErrorNewDeleteTypeMismatch error(GetCurrentTidOrInvalid(), free_stack, addr, + delete_size, delete_alignment); in_report.ReportError(error); } @@ -360,17 +298,58 @@ static NOINLINE void ReportInvalidPointerPair(uptr pc, uptr bp, uptr sp, in_report.ReportError(error); } +static bool IsInvalidPointerPair(uptr a1, uptr a2) { + if (a1 == a2) + return false; + + // 256B in shadow memory can be iterated quite fast + static const uptr kMaxOffset = 2048; + + uptr left = a1 < a2 ? a1 : a2; + uptr right = a1 < a2 ? a2 : a1; + uptr offset = right - left; + if (offset <= kMaxOffset) + return __asan_region_is_poisoned(left, offset); + + AsanThread *t = GetCurrentThread(); + + // check whether left is a stack memory pointer + if (uptr shadow_offset1 = t->GetStackVariableShadowStart(left)) { + uptr shadow_offset2 = t->GetStackVariableShadowStart(right); + return shadow_offset2 == 0 || shadow_offset1 != shadow_offset2; + } + + // check whether left is a heap memory address + HeapAddressDescription hdesc1, hdesc2; + if (GetHeapAddressInformation(left, 0, &hdesc1) && + hdesc1.chunk_access.access_type == kAccessTypeInside) + return !GetHeapAddressInformation(right, 0, &hdesc2) || + hdesc2.chunk_access.access_type != kAccessTypeInside || + hdesc1.chunk_access.chunk_begin != hdesc2.chunk_access.chunk_begin; + + // check whether left is an address of a global variable + GlobalAddressDescription gdesc1, gdesc2; + if (GetGlobalAddressInformation(left, 0, &gdesc1)) + return !GetGlobalAddressInformation(right - 1, 0, &gdesc2) || + !gdesc1.PointsInsideTheSameVariable(gdesc2); + + if (t->GetStackVariableShadowStart(right) || + GetHeapAddressInformation(right, 0, &hdesc2) || + GetGlobalAddressInformation(right - 1, 0, &gdesc2)) + return true; + + // At this point we know nothing about both a1 and a2 addresses. + return false; +} + static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) { if (!flags()->detect_invalid_pointer_pairs) return; uptr a1 = reinterpret_cast<uptr>(p1); uptr a2 = reinterpret_cast<uptr>(p2); - AsanChunkView chunk1 = FindHeapChunkByAddress(a1); - AsanChunkView chunk2 = FindHeapChunkByAddress(a2); - bool valid1 = chunk1.IsAllocated(); - bool valid2 = chunk2.IsAllocated(); - if (!valid1 || !valid2 || !chunk1.Eq(chunk2)) { + + if (IsInvalidPointerPair(a1, a2)) { GET_CALLER_PC_BP_SP; - return ReportInvalidPointerPair(pc, bp, sp, a1, a2); + ReportInvalidPointerPair(pc, bp, sp, a1, a2); } } // ----------------------- Mac-specific reports ----------------- {{{1 diff --git a/contrib/compiler-rt/lib/asan/asan_report.h b/contrib/compiler-rt/lib/asan/asan_report.h index 5a3533a319af..f2cdad47ee81 100644 --- a/contrib/compiler-rt/lib/asan/asan_report.h +++ b/contrib/compiler-rt/lib/asan/asan_report.h @@ -46,9 +46,9 @@ bool ParseFrameDescription(const char *frame_descr, // Different kinds of error reports. void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write, uptr access_size, u32 exp, bool fatal); -void ReportStackOverflow(const SignalContext &sig); -void ReportDeadlySignal(int signo, const SignalContext &sig); -void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size, +void ReportDeadlySignal(const SignalContext &sig); +void ReportNewDeleteTypeMismatch(uptr addr, uptr delete_size, + uptr delete_alignment, BufferedStackTrace *free_stack); void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack); void ReportFreeNotMalloced(uptr addr, BufferedStackTrace *free_stack); diff --git a/contrib/compiler-rt/lib/asan/asan_rtl.cc b/contrib/compiler-rt/lib/asan/asan_rtl.cc index 5ae3568ae04a..21fd0e240708 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtl.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtl.cc @@ -84,26 +84,6 @@ void ShowStatsAndAbort() { Die(); } -// ---------------------- mmap -------------------- {{{1 -// Reserve memory range [beg, end]. -// We need to use inclusive range because end+1 may not be representable. -void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { - CHECK_EQ((beg % GetMmapGranularity()), 0); - CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); - uptr size = end - beg + 1; - DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. - void *res = MmapFixedNoReserve(beg, size, name); - if (res != (void*)beg) { - Report("ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " - "Perhaps you're using ulimit -v\n", size); - Abort(); - } - if (common_flags()->no_huge_pages_for_shadow) - NoHugePagesInRegion(beg, size); - if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(beg, size); -} - // --------------- LowLevelAllocateCallbac ---------- {{{1 static void OnLowLevelAllocate(uptr ptr, uptr size) { PoisonShadow(ptr, size, kAsanInternalHeapMagic); @@ -327,7 +307,7 @@ static void asan_atexit() { static void InitializeHighMemEnd() { #if !ASAN_FIXED_MAPPING - kHighMemEnd = GetMaxVirtualAddress(); + kHighMemEnd = GetMaxUserVirtualAddress(); // Increase kHighMemEnd to make sure it's properly // aligned together with kHighMemBeg: kHighMemEnd |= SHADOW_GRANULARITY * GetMmapGranularity() - 1; @@ -335,46 +315,7 @@ static void InitializeHighMemEnd() { CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0); } -static void ProtectGap(uptr addr, uptr size) { - if (!flags()->protect_shadow_gap) { - // The shadow gap is unprotected, so there is a chance that someone - // is actually using this memory. Which means it needs a shadow... - uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached()); - uptr GapShadowEnd = - RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1; - if (Verbosity()) - Printf("protect_shadow_gap=0:" - " not protecting shadow gap, allocating gap's shadow\n" - "|| `[%p, %p]` || ShadowGap's shadow ||\n", GapShadowBeg, - GapShadowEnd); - ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd, - "unprotected gap shadow"); - return; - } - void *res = MmapFixedNoAccess(addr, size, "shadow gap"); - if (addr == (uptr)res) - return; - // A few pages at the start of the address space can not be protected. - // But we really want to protect as much as possible, to prevent this memory - // being returned as a result of a non-FIXED mmap(). - if (addr == kZeroBaseShadowStart) { - uptr step = GetMmapGranularity(); - while (size > step && addr < kZeroBaseMaxShadowStart) { - addr += step; - size -= step; - void *res = MmapFixedNoAccess(addr, size, "shadow gap"); - if (addr == (uptr)res) - return; - } - } - - Report("ERROR: Failed to protect the shadow gap. " - "ASan cannot proceed correctly. ABORTING.\n"); - DumpProcessMap(); - Die(); -} - -static void PrintAddressSpaceLayout() { +void PrintAddressSpaceLayout() { Printf("|| `[%p, %p]` || HighMem ||\n", (void*)kHighMemBeg, (void*)kHighMemEnd); Printf("|| `[%p, %p]` || HighShadow ||\n", @@ -426,71 +367,6 @@ static void PrintAddressSpaceLayout() { kHighShadowBeg > kMidMemEnd); } -static void InitializeShadowMemory() { - // Set the shadow memory address to uninitialized. - __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel; - - uptr shadow_start = kLowShadowBeg; - // Detect if a dynamic shadow address must used and find a available location - // when necessary. When dynamic address is used, the macro |kLowShadowBeg| - // expands to |__asan_shadow_memory_dynamic_address| which is - // |kDefaultShadowSentinel|. - if (shadow_start == kDefaultShadowSentinel) { - __asan_shadow_memory_dynamic_address = 0; - CHECK_EQ(0, kLowShadowBeg); - shadow_start = FindDynamicShadowStart(); - } - // Update the shadow memory address (potentially) used by instrumentation. - __asan_shadow_memory_dynamic_address = shadow_start; - - if (kLowShadowBeg) - shadow_start -= GetMmapGranularity(); - bool full_shadow_is_available = - MemoryRangeIsAvailable(shadow_start, kHighShadowEnd); - -#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) && \ - !ASAN_FIXED_MAPPING - if (!full_shadow_is_available) { - kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0; - kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0; - } -#endif - - if (Verbosity()) PrintAddressSpaceLayout(); - - if (full_shadow_is_available) { - // mmap the low shadow plus at least one page at the left. - if (kLowShadowBeg) - ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); - // mmap the high shadow. - ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); - // protect the gap. - ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); - CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); - } else if (kMidMemBeg && - MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) && - MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) { - CHECK(kLowShadowBeg != kLowShadowEnd); - // mmap the low shadow plus at least one page at the left. - ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); - // mmap the mid shadow. - ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow"); - // mmap the high shadow. - ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); - // protect the gaps. - ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); - ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1); - ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1); - } else { - Report("Shadow memory range interleaves with an existing memory mapping. " - "ASan cannot proceed correctly. ABORTING.\n"); - Report("ASan shadow was supposed to be located in the [%p-%p] range.\n", - shadow_start, kHighShadowEnd); - DumpProcessMap(); - Die(); - } -} - static void AsanInitInternal() { if (LIKELY(asan_inited)) return; SanitizerToolName = "AddressSanitizer"; @@ -531,6 +407,7 @@ static void AsanInitInternal() { MaybeReexec(); // Setup internal allocator callback. + SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); SetLowLevelAllocateCallback(OnLowLevelAllocate); InitializeAsanInterceptors(); @@ -575,20 +452,18 @@ static void AsanInitInternal() { InitTlsSize(); // Create main thread. - AsanThread *main_thread = AsanThread::Create( - /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0, - /* stack */ nullptr, /* detached */ true); + AsanThread *main_thread = CreateMainThread(); CHECK_EQ(0, main_thread->tid()); - SetCurrentThread(main_thread); - main_thread->ThreadStart(internal_getpid(), - /* signal_thread_is_registered */ nullptr); force_interface_symbols(); // no-op. SanitizerInitializeUnwinder(); if (CAN_SANITIZE_LEAKS) { __lsan::InitCommonLsan(); if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) { - Atexit(__lsan::DoLeakCheck); + if (flags()->halt_on_error) + Atexit(__lsan::DoLeakCheck); + else + Atexit(__lsan::DoRecoverableLeakCheckVoid); } } @@ -608,6 +483,11 @@ static void AsanInitInternal() { } VReport(1, "AddressSanitizer Init done\n"); + + if (flags()->sleep_after_init) { + Report("Sleeping for %d second(s)\n", flags()->sleep_after_init); + SleepForSeconds(flags()->sleep_after_init); + } } // Initialize as requested from some part of ASan runtime library (interceptors, @@ -647,6 +527,7 @@ void NOINLINE __asan_handle_no_return() { top = curr_thread->stack_top(); bottom = ((uptr)&local_stack - PageSize) & ~(PageSize - 1); } else { + CHECK(!SANITIZER_FUCHSIA); // If we haven't seen this thread, try asking the OS for stack bounds. uptr tls_addr, tls_size, stack_size; GetThreadStackAndTls(/*main=*/false, &bottom, &stack_size, &tls_addr, diff --git a/contrib/compiler-rt/lib/asan/asan_scariness_score.h b/contrib/compiler-rt/lib/asan/asan_scariness_score.h index 7f1571416fd5..7f095dd29f29 100644 --- a/contrib/compiler-rt/lib/asan/asan_scariness_score.h +++ b/contrib/compiler-rt/lib/asan/asan_scariness_score.h @@ -47,7 +47,7 @@ struct ScarinessScoreBase { }; int GetScore() const { return score; } const char *GetDescription() const { return descr; } - void Print() { + void Print() const { if (score && flags()->print_scariness) Printf("SCARINESS: %d (%s)\n", score, descr); } diff --git a/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc b/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc new file mode 100644 index 000000000000..b3cf0b5c1ce2 --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc @@ -0,0 +1,165 @@ +//===-- asan_shadow_setup.cc ----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Set up the shadow memory. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" + +// asan_fuchsia.cc has its own InitializeShadowMemory implementation. +#if !SANITIZER_FUCHSIA + +#include "asan_internal.h" +#include "asan_mapping.h" + +namespace __asan { + +// ---------------------- mmap -------------------- {{{1 +// Reserve memory range [beg, end]. +// We need to use inclusive range because end+1 may not be representable. +void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { + CHECK_EQ((beg % GetMmapGranularity()), 0); + CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); + uptr size = end - beg + 1; + DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. + void *res = MmapFixedNoReserve(beg, size, name); + if (res != (void *)beg) { + Report( + "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " + "Perhaps you're using ulimit -v\n", + size); + Abort(); + } + if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size); +} + +static void ProtectGap(uptr addr, uptr size) { + if (!flags()->protect_shadow_gap) { + // The shadow gap is unprotected, so there is a chance that someone + // is actually using this memory. Which means it needs a shadow... + uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached()); + uptr GapShadowEnd = + RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1; + if (Verbosity()) + Printf( + "protect_shadow_gap=0:" + " not protecting shadow gap, allocating gap's shadow\n" + "|| `[%p, %p]` || ShadowGap's shadow ||\n", + GapShadowBeg, GapShadowEnd); + ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd, + "unprotected gap shadow"); + return; + } + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + // A few pages at the start of the address space can not be protected. + // But we really want to protect as much as possible, to prevent this memory + // being returned as a result of a non-FIXED mmap(). + if (addr == kZeroBaseShadowStart) { + uptr step = GetMmapGranularity(); + while (size > step && addr < kZeroBaseMaxShadowStart) { + addr += step; + size -= step; + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + } + } + + Report( + "ERROR: Failed to protect the shadow gap. " + "ASan cannot proceed correctly. ABORTING.\n"); + DumpProcessMap(); + Die(); +} + +static void MaybeReportLinuxPIEBug() { +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__aarch64__)) + Report("This might be related to ELF_ET_DYN_BASE change in Linux 4.12.\n"); + Report( + "See https://github.com/google/sanitizers/issues/856 for possible " + "workarounds.\n"); +#endif +} + +void InitializeShadowMemory() { + // Set the shadow memory address to uninitialized. + __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel; + + uptr shadow_start = kLowShadowBeg; + // Detect if a dynamic shadow address must used and find a available location + // when necessary. When dynamic address is used, the macro |kLowShadowBeg| + // expands to |__asan_shadow_memory_dynamic_address| which is + // |kDefaultShadowSentinel|. + bool full_shadow_is_available = false; + if (shadow_start == kDefaultShadowSentinel) { + __asan_shadow_memory_dynamic_address = 0; + CHECK_EQ(0, kLowShadowBeg); + shadow_start = FindDynamicShadowStart(); + if (SANITIZER_LINUX) full_shadow_is_available = true; + } + // Update the shadow memory address (potentially) used by instrumentation. + __asan_shadow_memory_dynamic_address = shadow_start; + + if (kLowShadowBeg) shadow_start -= GetMmapGranularity(); + + if (!full_shadow_is_available) + full_shadow_is_available = + MemoryRangeIsAvailable(shadow_start, kHighShadowEnd); + +#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) && \ + !ASAN_FIXED_MAPPING + if (!full_shadow_is_available) { + kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0; + kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0; + } +#endif + + if (Verbosity()) PrintAddressSpaceLayout(); + + if (full_shadow_is_available) { + // mmap the low shadow plus at least one page at the left. + if (kLowShadowBeg) + ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); + // mmap the high shadow. + ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); + // protect the gap. + ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); + CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); + } else if (kMidMemBeg && + MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) && + MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) { + CHECK(kLowShadowBeg != kLowShadowEnd); + // mmap the low shadow plus at least one page at the left. + ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); + // mmap the mid shadow. + ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow"); + // mmap the high shadow. + ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); + // protect the gaps. + ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); + ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1); + ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1); + } else { + Report( + "Shadow memory range interleaves with an existing memory mapping. " + "ASan cannot proceed correctly. ABORTING.\n"); + Report("ASan shadow was supposed to be located in the [%p-%p] range.\n", + shadow_start, kHighShadowEnd); + MaybeReportLinuxPIEBug(); + DumpProcessMap(); + Die(); + } +} + +} // namespace __asan + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_stack.h b/contrib/compiler-rt/lib/asan/asan_stack.h index cc95e0f30a33..8e9df888d798 100644 --- a/contrib/compiler-rt/lib/asan/asan_stack.h +++ b/contrib/compiler-rt/lib/asan/asan_stack.h @@ -31,9 +31,8 @@ u32 GetMallocContextSize(); // The pc will be in the position 0 of the resulting stack trace. // The bp may refer to the current frame or to the caller's frame. ALWAYS_INLINE -void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, - uptr pc, uptr bp, void *context, - bool fast) { +void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, + void *context, bool fast) { #if SANITIZER_WINDOWS stack->Unwind(max_depth, pc, bp, context, 0, 0, fast); #else @@ -41,10 +40,6 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, stack->size = 0; if (LIKELY(asan_inited)) { if ((t = GetCurrentThread()) && !t->isUnwinding()) { - // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace() - // yields the call stack of the signal's handler and not of the code - // that raised the signal (as it does on Linux). - if (SANITIZER_FREEBSD && t->isInDeadlySignal()) fast = true; uptr stack_top = t->stack_top(); uptr stack_bottom = t->stack_bottom(); ScopedUnwinding unwind_scope(t); @@ -66,32 +61,29 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, // as early as possible (in functions exposed to the user), as we generally // don't want stack trace to contain functions from ASan internals. -#define GET_STACK_TRACE(max_size, fast) \ - BufferedStackTrace stack; \ - if (max_size <= 2) { \ - stack.size = max_size; \ - if (max_size > 0) { \ - stack.top_frame_bp = GET_CURRENT_FRAME(); \ - stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ - if (max_size > 1) \ - stack.trace_buffer[1] = GET_CALLER_PC(); \ - } \ - } else { \ - GetStackTraceWithPcBpAndContext(&stack, max_size, \ - StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), 0, fast); \ +#define GET_STACK_TRACE(max_size, fast) \ + BufferedStackTrace stack; \ + if (max_size <= 2) { \ + stack.size = max_size; \ + if (max_size > 0) { \ + stack.top_frame_bp = GET_CURRENT_FRAME(); \ + stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ + if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \ + } \ + } else { \ + GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), 0, fast); \ } -#define GET_STACK_TRACE_FATAL(pc, bp) \ - BufferedStackTrace stack; \ - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, 0, \ - common_flags()->fast_unwind_on_fatal) +#define GET_STACK_TRACE_FATAL(pc, bp) \ + BufferedStackTrace stack; \ + GetStackTrace(&stack, kStackTraceMax, pc, bp, 0, \ + common_flags()->fast_unwind_on_fatal) -#define GET_STACK_TRACE_SIGNAL(sig) \ - BufferedStackTrace stack; \ - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, \ - (sig).pc, (sig).bp, (sig).context, \ - common_flags()->fast_unwind_on_fatal) +#define GET_STACK_TRACE_SIGNAL(sig) \ + BufferedStackTrace stack; \ + GetStackTrace(&stack, kStackTraceMax, (sig).pc, (sig).bp, (sig).context, \ + common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_FATAL_HERE \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) diff --git a/contrib/compiler-rt/lib/asan/asan_thread.cc b/contrib/compiler-rt/lib/asan/asan_thread.cc index b1a0d9a3b37f..ad81512dff08 100644 --- a/contrib/compiler-rt/lib/asan/asan_thread.cc +++ b/contrib/compiler-rt/lib/asan/asan_thread.cc @@ -27,11 +27,6 @@ namespace __asan { // AsanThreadContext implementation. -struct CreateThreadContextArgs { - AsanThread *thread; - StackTrace *stack; -}; - void AsanThreadContext::OnCreated(void *arg) { CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs*>(arg); if (args->stack) @@ -88,7 +83,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg, AsanThread *thread = (AsanThread*)MmapOrDie(size, __func__); thread->start_routine_ = start_routine; thread->arg_ = arg; - CreateThreadContextArgs args = { thread, stack }; + AsanThreadContext::CreateThreadContextArgs args = {thread, stack}; asanThreadRegistry().CreateThread(*reinterpret_cast<uptr *>(thread), detached, parent_tid, &args); @@ -223,12 +218,12 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() { return nullptr; } -void AsanThread::Init() { +void AsanThread::Init(const InitOptions *options) { next_stack_top_ = next_stack_bottom_ = 0; atomic_store(&stack_switching_, false, memory_order_release); fake_stack_ = nullptr; // Will be initialized lazily if needed. CHECK_EQ(this->stack_size(), 0U); - SetThreadStackAndTls(); + SetThreadStackAndTls(options); CHECK_GT(this->stack_size(), 0U); CHECK(AddrIsInMem(stack_bottom_)); CHECK(AddrIsInMem(stack_top_ - 1)); @@ -239,6 +234,10 @@ void AsanThread::Init() { &local); } +// Fuchsia doesn't use ThreadStart. +// asan_fuchsia.c defines CreateMainThread and SetThreadStackAndTls. +#if !SANITIZER_FUCHSIA + thread_return_t AsanThread::ThreadStart( tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) { Init(); @@ -270,7 +269,21 @@ thread_return_t AsanThread::ThreadStart( return res; } -void AsanThread::SetThreadStackAndTls() { +AsanThread *CreateMainThread() { + AsanThread *main_thread = AsanThread::Create( + /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0, + /* stack */ nullptr, /* detached */ true); + SetCurrentThread(main_thread); + main_thread->ThreadStart(internal_getpid(), + /* signal_thread_is_registered */ nullptr); + return main_thread; +} + +// This implementation doesn't use the argument, which is just passed down +// from the caller of Init (which see, above). It's only there to support +// OS-specific implementations that need more information passed through. +void AsanThread::SetThreadStackAndTls(const InitOptions *options) { + DCHECK_EQ(options, nullptr); uptr tls_size = 0; uptr stack_size = 0; GetThreadStackAndTls(tid() == 0, const_cast<uptr *>(&stack_bottom_), @@ -283,6 +296,8 @@ void AsanThread::SetThreadStackAndTls() { CHECK(AddrIsInStack((uptr)&local)); } +#endif // !SANITIZER_FUCHSIA + void AsanThread::ClearShadowForThreadStackAndTLS() { PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0); if (tls_begin_ != tls_end_) @@ -302,7 +317,7 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, access->frame_descr = (const char *)((uptr*)bottom)[1]; return true; } - uptr aligned_addr = addr & ~(SANITIZER_WORDSIZE/8 - 1); // align addr. + uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. uptr mem_ptr = RoundDownTo(aligned_addr, SHADOW_GRANULARITY); u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); u8 *shadow_bottom = (u8*)MemToShadow(bottom); @@ -331,6 +346,29 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, return true; } +uptr AsanThread::GetStackVariableShadowStart(uptr addr) { + uptr bottom = 0; + if (AddrIsInStack(addr)) { + bottom = stack_bottom(); + } else if (has_fake_stack()) { + bottom = fake_stack()->AddrIsInFakeStack(addr); + CHECK(bottom); + } else + return 0; + + uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. + u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); + u8 *shadow_bottom = (u8*)MemToShadow(bottom); + + while (shadow_ptr >= shadow_bottom && + (*shadow_ptr != kAsanStackLeftRedzoneMagic && + *shadow_ptr != kAsanStackMidRedzoneMagic && + *shadow_ptr != kAsanStackRightRedzoneMagic)) + shadow_ptr--; + + return (uptr)shadow_ptr + 1; +} + bool AsanThread::AddrIsInStack(uptr addr) { const auto bounds = GetStackBounds(); return addr >= bounds.bottom && addr < bounds.top; diff --git a/contrib/compiler-rt/lib/asan/asan_thread.h b/contrib/compiler-rt/lib/asan/asan_thread.h index 424f9e68dfea..66091921101f 100644 --- a/contrib/compiler-rt/lib/asan/asan_thread.h +++ b/contrib/compiler-rt/lib/asan/asan_thread.h @@ -49,6 +49,11 @@ class AsanThreadContext : public ThreadContextBase { void OnCreated(void *arg) override; void OnFinished() override; + + struct CreateThreadContextArgs { + AsanThread *thread; + StackTrace *stack; + }; }; // AsanThreadContext objects are never freed, so we need many of them. @@ -62,7 +67,9 @@ class AsanThread { static void TSDDtor(void *tsd); void Destroy(); - void Init(); // Should be called from the thread itself. + struct InitOptions; + void Init(const InitOptions *options = nullptr); + thread_return_t ThreadStart(tid_t os_id, atomic_uintptr_t *signal_thread_is_registered); @@ -83,6 +90,9 @@ class AsanThread { }; bool GetStackFrameAccessByAddr(uptr addr, StackFrameAccess *access); + // Returns a pointer to the start of the stack variable's shadow memory. + uptr GetStackVariableShadowStart(uptr addr); + bool AddrIsInStack(uptr addr); void DeleteFakeStack(int tid) { @@ -118,17 +128,15 @@ class AsanThread { bool isUnwinding() const { return unwinding_; } void setUnwinding(bool b) { unwinding_ = b; } - // True if we are in a deadly signal handler. - bool isInDeadlySignal() const { return in_deadly_signal_; } - void setInDeadlySignal(bool b) { in_deadly_signal_ = b; } - AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } AsanStats &stats() { return stats_; } private: // NOTE: There is no AsanThread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. - void SetThreadStackAndTls(); + + void SetThreadStackAndTls(const InitOptions *options); + void ClearShadowForThreadStackAndTLS(); FakeStack *AsyncSignalSafeLazyInitFakeStack(); @@ -158,7 +166,6 @@ class AsanThread { AsanThreadLocalMallocStorage malloc_storage_; AsanStats stats_; bool unwinding_; - bool in_deadly_signal_; }; // ScopedUnwinding is a scope for stacktracing member of a context @@ -173,20 +180,6 @@ class ScopedUnwinding { AsanThread *thread; }; -// ScopedDeadlySignal is a scope for handling deadly signals. -class ScopedDeadlySignal { - public: - explicit ScopedDeadlySignal(AsanThread *t) : thread(t) { - if (thread) thread->setInDeadlySignal(true); - } - ~ScopedDeadlySignal() { - if (thread) thread->setInDeadlySignal(false); - } - - private: - AsanThread *thread; -}; - // Returns a single instance of registry. ThreadRegistry &asanThreadRegistry(); diff --git a/contrib/compiler-rt/lib/asan/asan_win.cc b/contrib/compiler-rt/lib/asan/asan_win.cc index 8a839d913f95..68eedd1863bc 100644 --- a/contrib/compiler-rt/lib/asan/asan_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_win.cc @@ -57,8 +57,8 @@ long __asan_unhandled_exception_filter(EXCEPTION_POINTERS *info) { // FIXME: Handle EXCEPTION_STACK_OVERFLOW here. - SignalContext sig = SignalContext::Create(exception_record, context); - ReportDeadlySignal(exception_record->ExceptionCode, sig); + SignalContext sig(exception_record, context); + ReportDeadlySignal(sig); UNREACHABLE("returned from reporting deadly signal"); } diff --git a/contrib/compiler-rt/lib/builtins/adddf3.c b/contrib/compiler-rt/lib/builtins/adddf3.c index c528e9e21f51..9a3901312e51 100644 --- a/contrib/compiler-rt/lib/builtins/adddf3.c +++ b/contrib/compiler-rt/lib/builtins/adddf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI double __adddf3(double a, double b){ } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_dadd(double a, double b) { return __adddf3(a, b); } +#else +AEABI_RTABI double __aeabi_dadd(double a, double b) COMPILER_RT_ALIAS(__adddf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/addsf3.c b/contrib/compiler-rt/lib/builtins/addsf3.c index fe570687a25e..c5c1a41c3611 100644 --- a/contrib/compiler-rt/lib/builtins/addsf3.c +++ b/contrib/compiler-rt/lib/builtins/addsf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI float __addsf3(float a, float b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_fadd(float a, float b) { return __addsf3(a, b); } +#else +AEABI_RTABI float __aeabi_fadd(float a, float b) COMPILER_RT_ALIAS(__addsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S index 3e7a8b86b739..87dd03dce94d 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S @@ -30,7 +30,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) push {r0-r3, lr} bl __aeabi_cdcmpeq_check_nan cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) beq 1f // NaN has been ruled out, so __aeabi_cdcmple can't trap mov r0, sp @@ -46,9 +46,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) pop {r0-r3, lr} // NaN has been ruled out, so __aeabi_cdcmple can't trap + // Use "it ne" + unconditional branch to guarantee a supported relocation if + // __aeabi_cdcmple is in a different section for some builds. + IT(ne) bne __aeabi_cdcmple -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) +#if defined(USE_THUMB_2) mov ip, #APSR_C msr APSR_nzcvq, ip #else @@ -78,7 +81,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple) bl __aeabi_dcmplt cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) bne 1f // Z = 0, C = 0 movs r0, #1 diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S index 1f304ffd964c..c5fee6b6a08e 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S @@ -30,7 +30,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) push {r0-r3, lr} bl __aeabi_cfcmpeq_check_nan cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) beq 1f // NaN has been ruled out, so __aeabi_cfcmple can't trap mov r0, sp @@ -46,9 +46,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) pop {r0-r3, lr} // NaN has been ruled out, so __aeabi_cfcmple can't trap + // Use "it ne" + unconditional branch to guarantee a supported relocation if + // __aeabi_cfcmple is in a different section for some builds. + IT(ne) bne __aeabi_cfcmple -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) +#if defined(USE_THUMB_2) mov ip, #APSR_C msr APSR_nzcvq, ip #else @@ -78,7 +81,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple) bl __aeabi_fcmplt cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) bne 1f // Z = 0, C = 0 movs r0, #1 diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S index 4419929f62ac..9c9c80ab5a7b 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S @@ -20,17 +20,10 @@ #endif .syntax unified - .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__aeabi_idivmod) -#else DEFINE_COMPILERRT_FUNCTION(__aeabi_idivmod) -#endif #if defined(USE_THUMB_1) push {r0, r1, lr} bl SYMBOL_NAME(__divsi3) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S index 33ea54848b26..e86d6113760e 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S @@ -14,7 +14,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memcmp) +#ifdef USE_THUMB_1 + push {r7, lr} + bl memcmp + pop {r7, pc} +#else b memcmp +#endif END_COMPILERRT_FUNCTION(__aeabi_memcmp) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcmp4, __aeabi_memcmp) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S index eabfa490494c..e83c5fd4dbb3 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S @@ -14,7 +14,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memcpy) +#ifdef USE_THUMB_1 + push {r7, lr} + bl memcpy + pop {r7, pc} +#else b memcpy +#endif END_COMPILERRT_FUNCTION(__aeabi_memcpy) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcpy4, __aeabi_memcpy) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S index 1bf08c0d5b75..ee28300e46f2 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S @@ -13,7 +13,13 @@ .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memmove) +#ifdef USE_THUMB_1 + push {r7, lr} + bl memmove + pop {r7, pc} +#else b memmove +#endif END_COMPILERRT_FUNCTION(__aeabi_memmove) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memmove4, __aeabi_memmove) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S index 633f592279b5..0a678d7627e7 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S @@ -18,16 +18,29 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_memset) mov r3, r1 mov r1, r2 mov r2, r3 +#ifdef USE_THUMB_1 + push {r7, lr} + bl memset + pop {r7, pc} +#else b memset +#endif END_COMPILERRT_FUNCTION(__aeabi_memset) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset4, __aeabi_memset) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset8, __aeabi_memset) + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memclr) mov r2, r1 movs r1, #0 +#ifdef USE_THUMB_1 + push {r7, lr} + bl memset + pop {r7, pc} +#else b memset +#endif END_COMPILERRT_FUNCTION(__aeabi_memclr) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memclr4, __aeabi_memclr) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S index 37dae4a105cc..88a4a6d8bc12 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S @@ -22,16 +22,10 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__aeabi_uidivmod) -#else DEFINE_COMPILERRT_FUNCTION(__aeabi_uidivmod) -#endif -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) cmp r0, r1 bcc LOCAL_LABEL(case_denom_larger) push {r0, r1, lr} @@ -44,7 +38,7 @@ LOCAL_LABEL(case_denom_larger): movs r1, r0 movs r0, #0 JMP (lr) -#else +#else // defined(USE_THUMB_1) push { lr } sub sp, sp, #4 mov r2, sp diff --git a/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S b/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S index 4e5a579cf16a..e9db8bac7994 100644 --- a/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S @@ -11,9 +11,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE // // extern uint64_t __bswapdi2(uint64_t); @@ -21,11 +19,7 @@ // Reverse all the bytes in a 64-bit integer. // .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__bswapdi2) -#else DEFINE_COMPILERRT_FUNCTION(__bswapdi2) -#endif #if __ARM_ARCH < 6 // before armv6 does not have "rev" instruction // r2 = rev(r0) diff --git a/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S b/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S index 60342ae0f2fb..1f6eed5c1bbf 100644 --- a/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S @@ -11,9 +11,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE // // extern uint32_t __bswapsi2(uint32_t); @@ -21,11 +19,7 @@ // Reverse all the bytes in a 32-bit integer. // .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__bswapsi2) -#else DEFINE_COMPILERRT_FUNCTION(__bswapsi2) -#endif #if __ARM_ARCH < 6 // before armv6 does not have "rev" instruction eor r1, r0, r0, ror #16 diff --git a/contrib/compiler-rt/lib/builtins/arm/clzdi2.S b/contrib/compiler-rt/lib/builtins/arm/clzdi2.S index fe56a183f1b1..fc03b385cdfa 100644 --- a/contrib/compiler-rt/lib/builtins/arm/clzdi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/clzdi2.S @@ -15,17 +15,10 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif - + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__clzdi2) -#else DEFINE_COMPILERRT_FUNCTION(__clzdi2) -#endif #ifdef __ARM_FEATURE_CLZ #ifdef __ARMEB__ cmp r0, 0 diff --git a/contrib/compiler-rt/lib/builtins/arm/clzsi2.S b/contrib/compiler-rt/lib/builtins/arm/clzsi2.S index 28d43041eaf2..f2ce59c90119 100644 --- a/contrib/compiler-rt/lib/builtins/arm/clzsi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/clzsi2.S @@ -15,16 +15,10 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__clzsi2) -#else DEFINE_COMPILERRT_FUNCTION(__clzsi2) -#endif #ifdef __ARM_FEATURE_CLZ clz r0, r0 JMP(lr) diff --git a/contrib/compiler-rt/lib/builtins/arm/comparesf2.S b/contrib/compiler-rt/lib/builtins/arm/comparesf2.S index 1f7031cbf665..c6c4cc067f07 100644 --- a/contrib/compiler-rt/lib/builtins/arm/comparesf2.S +++ b/contrib/compiler-rt/lib/builtins/arm/comparesf2.S @@ -40,25 +40,19 @@ #include "../assembly.h" .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ int __eqsf2(float a, float b) .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__eqsf2) -#else DEFINE_COMPILERRT_FUNCTION(__eqsf2) -#endif #if defined(COMPILER_RT_ARMHF_TARGET) vmov r0, s0 vmov r1, s1 #endif // Make copies of a and b with the sign bit shifted off the top. These will // be used to detect zeros and NaNs. -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) push {r6, lr} lsls r2, r0, #1 lsls r3, r1, #1 @@ -150,7 +144,7 @@ DEFINE_COMPILERRT_FUNCTION(__eqsf2) // If a == b, then the Z flag is set, so we can get the correct final value // into r0 by simply or'ing with 1 if Z is clear. // For Thumb-1, r0 contains -1 if a < b, 0 if a > b and 0 if a == b. -#if defined(USE_THUMB_1) +#if !defined(USE_THUMB_1) it ne orrne r0, r0, #1 #endif @@ -185,11 +179,7 @@ DEFINE_COMPILERRT_FUNCTION_ALIAS(__nesf2, __eqsf2) @ int __gtsf2(float a, float b) .p2align 2 -#if defined(USE_THUMB) -DEFINE_COMPILERRT_THUMB_FUNCTION(__gtsf2) -#else DEFINE_COMPILERRT_FUNCTION(__gtsf2) -#endif // Identical to the preceding except in that we return -1 for NaN values. // Given that the two paths share so much code, one might be tempted to // unify them; however, the extra code needed to do so makes the code size @@ -263,11 +253,7 @@ DEFINE_COMPILERRT_FUNCTION_ALIAS(__gesf2, __gtsf2) @ int __unordsf2(float a, float b) .p2align 2 -#if defined(USE_THUMB) -DEFINE_COMPILERRT_THUMB_FUNCTION(__unordsf2) -#else DEFINE_COMPILERRT_FUNCTION(__unordsf2) -#endif #if defined(COMPILER_RT_ARMHF_TARGET) vmov r0, s0 diff --git a/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S b/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S index 3c8359852009..8a027b741efe 100644 --- a/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S +++ b/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S @@ -23,20 +23,14 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ int __divmodsi4(int divident, int divisor, int *remainder) @ Calculate the quotient and remainder of the (signed) division. The return @ value is the quotient, the remainder is placed in the variable. .p2align 3 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__divmodsi4) -#else DEFINE_COMPILERRT_FUNCTION(__divmodsi4) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divzero) diff --git a/contrib/compiler-rt/lib/builtins/arm/divsi3.S b/contrib/compiler-rt/lib/builtins/arm/divsi3.S index ad96c9e5bfa9..19757af177eb 100644 --- a/contrib/compiler-rt/lib/builtins/arm/divsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/divsi3.S @@ -22,9 +22,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 3 // Ok, APCS and AAPCS agree on 32 bit args, so it's safe to use the same routine. @@ -33,11 +31,7 @@ DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_idiv, __divsi3) @ int __divsi3(int divident, int divisor) @ Calculate and return the quotient of the (signed) division. -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__divsi3) -#else DEFINE_COMPILERRT_FUNCTION(__divsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1,r1 beq LOCAL_LABEL(divzero) @@ -49,14 +43,14 @@ LOCAL_LABEL(divzero): #else ESTABLISH_FRAME // Set aside the sign of the quotient. -# if __ARM_ARCH_ISA_THUMB == 1 +# if defined(USE_THUMB_1) movs r4, r0 eors r4, r1 # else eor r4, r0, r1 # endif // Take absolute value of a and b via abs(x) = (x^(x >> 31)) - (x >> 31). -# if __ARM_ARCH_ISA_THUMB == 1 +# if defined(USE_THUMB_1) asrs r2, r0, #31 asrs r3, r1, #31 eors r0, r2 diff --git a/contrib/compiler-rt/lib/builtins/arm/modsi3.S b/contrib/compiler-rt/lib/builtins/arm/modsi3.S index 0986d84945a6..be263834d7f1 100644 --- a/contrib/compiler-rt/lib/builtins/arm/modsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/modsi3.S @@ -22,19 +22,13 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ int __modsi3(int divident, int divisor) @ Calculate and return the remainder of the (signed) division. .p2align 3 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__modsi3) -#else DEFINE_COMPILERRT_FUNCTION(__modsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divzero) diff --git a/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S b/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S index 860fced63085..ee3950c9b0eb 100644 --- a/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S +++ b/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S @@ -16,9 +16,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ unsigned int __udivmodsi4(unsigned int divident, unsigned int divisor, @ unsigned int *remainder) @@ -26,11 +24,7 @@ @ value is the quotient, the remainder is placed in the variable. .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__udivmodsi4) -#else DEFINE_COMPILERRT_FUNCTION(__udivmodsi4) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divby0) diff --git a/contrib/compiler-rt/lib/builtins/arm/udivsi3.S b/contrib/compiler-rt/lib/builtins/arm/udivsi3.S index 82db2cbb395f..6dea27d404ff 100644 --- a/contrib/compiler-rt/lib/builtins/arm/udivsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/udivsi3.S @@ -16,9 +16,8 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + +DEFINE_CODE_STATE .p2align 2 DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_uidiv, __udivsi3) @@ -26,11 +25,7 @@ DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_uidiv, __udivsi3) @ unsigned int __udivsi3(unsigned int divident, unsigned int divisor) @ Calculate and return the quotient of the (unsigned) division. -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__udivsi3) -#else DEFINE_COMPILERRT_FUNCTION(__udivsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divby0) diff --git a/contrib/compiler-rt/lib/builtins/arm/umodsi3.S b/contrib/compiler-rt/lib/builtins/arm/umodsi3.S index 90f3e8575c77..069fad34cb9c 100644 --- a/contrib/compiler-rt/lib/builtins/arm/umodsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/umodsi3.S @@ -16,19 +16,13 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ unsigned int __umodsi3(unsigned int divident, unsigned int divisor) @ Calculate and return the remainder of the (unsigned) division. .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__umodsi3) -#else DEFINE_COMPILERRT_FUNCTION(__umodsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divby0) diff --git a/contrib/compiler-rt/lib/builtins/ashldi3.c b/contrib/compiler-rt/lib/builtins/ashldi3.c index fcb0abdb1fce..a5c1836006b9 100644 --- a/contrib/compiler-rt/lib/builtins/ashldi3.c +++ b/contrib/compiler-rt/lib/builtins/ashldi3.c @@ -41,8 +41,5 @@ __ashldi3(di_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) { - return __ashldi3(a, b); -} +AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) COMPILER_RT_ALIAS(__ashldi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/ashrdi3.c b/contrib/compiler-rt/lib/builtins/ashrdi3.c index b4ab4c617ba0..84619965eca0 100644 --- a/contrib/compiler-rt/lib/builtins/ashrdi3.c +++ b/contrib/compiler-rt/lib/builtins/ashrdi3.c @@ -42,8 +42,5 @@ __ashrdi3(di_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) { - return __ashrdi3(a, b); -} +AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) COMPILER_RT_ALIAS(__ashrdi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/assembly.h b/contrib/compiler-rt/lib/builtins/assembly.h index 4893d483fd82..3f5e59b25442 100644 --- a/contrib/compiler-rt/lib/builtins/assembly.h +++ b/contrib/compiler-rt/lib/builtins/assembly.h @@ -68,10 +68,42 @@ #endif #if defined(__arm__) + +/* + * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: + * - for '-mthumb -march=armv6' compiler defines '__thumb__' + * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' + */ +#if defined(__thumb2__) || defined(__thumb__) +#define DEFINE_CODE_STATE .thumb SEPARATOR +#define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR +#if defined(__thumb2__) +#define USE_THUMB_2 +#define IT(cond) it cond +#define ITT(cond) itt cond +#define ITE(cond) ite cond +#else +#define USE_THUMB_1 +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif // defined(__thumb__2) +#else // !defined(__thumb2__) && !defined(__thumb__) +#define DEFINE_CODE_STATE .arm SEPARATOR +#define DECLARE_FUNC_ENCODING +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif + +#if defined(USE_THUMB_1) && defined(USE_THUMB_2) +#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together." +#endif + #if defined(__ARM_ARCH_4T__) || __ARM_ARCH >= 5 #define ARM_HAS_BX #endif -#if !defined(__ARM_FEATURE_CLZ) && __ARM_ARCH_ISA_THUMB != 1 && \ +#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) && \ (__ARM_ARCH >= 6 || (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__))) #define __ARM_FEATURE_CLZ #endif @@ -93,37 +125,15 @@ JMP(ip) #endif -/* - * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: - * - for '-mthumb -march=armv6' compiler defines '__thumb__' - * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' - */ -#if defined(__thumb2__) -#define USE_THUMB_2 1 -#elif defined(__thumb__) -#define USE_THUMB_1 1 -#endif - -#if defined(USE_THUMB_1) && defined(USE_THUMB_2) -#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together." -#endif - -#if defined(USE_THUMB_1) || defined(USE_THUMB_1) -#define USE_THUMB_PROLOGUE 1 -#endif - #if defined(USE_THUMB_2) -#define IT(cond) it cond -#define ITT(cond) itt cond -#define ITE(cond) ite cond #define WIDE(op) op.w #else -#define IT(cond) -#define ITT(cond) -#define ITE(cond) #define WIDE(op) op #endif -#endif /* defined(__arm__) */ +#else // !defined(__arm) +#define DECLARE_FUNC_ENCODING +#define DEFINE_CODE_STATE +#endif #define GLUE2(a, b) a##b #define GLUE(a, b) GLUE2(a, b) @@ -137,13 +147,16 @@ #endif #define DEFINE_COMPILERRT_FUNCTION(name) \ + DEFINE_CODE_STATE \ FILE_LEVEL_DIRECTIVE SEPARATOR \ .globl SYMBOL_NAME(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ DECLARE_SYMBOL_VISIBILITY(name) \ + DECLARE_FUNC_ENCODING \ SYMBOL_NAME(name): #define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \ + DEFINE_CODE_STATE \ FILE_LEVEL_DIRECTIVE SEPARATOR \ .globl SYMBOL_NAME(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ @@ -152,16 +165,20 @@ SYMBOL_NAME(name): #define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \ + DEFINE_CODE_STATE \ FILE_LEVEL_DIRECTIVE SEPARATOR \ .globl SYMBOL_NAME(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ HIDDEN(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_FUNC_ENCODING \ SYMBOL_NAME(name): #define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \ + DEFINE_CODE_STATE \ .globl name SEPARATOR \ SYMBOL_IS_FUNC(name) SEPARATOR \ HIDDEN(name) SEPARATOR \ + DECLARE_FUNC_ENCODING \ name: #define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \ diff --git a/contrib/compiler-rt/lib/builtins/clear_cache.c b/contrib/compiler-rt/lib/builtins/clear_cache.c index 7ccbe019dba9..4a01cb46d4ac 100644 --- a/contrib/compiler-rt/lib/builtins/clear_cache.c +++ b/contrib/compiler-rt/lib/builtins/clear_cache.c @@ -9,6 +9,7 @@ */ #include "int_lib.h" +#include <assert.h> #include <stddef.h> #if __APPLE__ @@ -23,7 +24,7 @@ uint32_t FlushInstructionCache(uintptr_t hProcess, void *lpBaseAddress, uintptr_t GetCurrentProcess(void); #endif -#if (defined(__FreeBSD__) || defined(__Bitrig__)) && defined(__arm__) +#if defined(__FreeBSD__) && defined(__arm__) #include <sys/types.h> #include <machine/sysarch.h> #endif @@ -32,7 +33,7 @@ uintptr_t GetCurrentProcess(void); #include <machine/sysarch.h> #endif -#if defined(__mips__) && !defined(__FreeBSD__) +#if defined(__linux__) && defined(__mips__) #include <sys/cachectl.h> #include <sys/syscall.h> #include <unistd.h> @@ -41,7 +42,7 @@ uintptr_t GetCurrentProcess(void); * clear_mips_cache - Invalidates instruction cache for Mips. */ static void clear_mips_cache(const void* Addr, size_t Size) { - asm volatile ( + __asm__ volatile ( ".set push\n" ".set noreorder\n" ".set noat\n" @@ -96,7 +97,7 @@ void __clear_cache(void *start, void *end) { * so there is nothing to do */ #elif defined(__arm__) && !defined(__APPLE__) - #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__Bitrig__) + #if defined(__FreeBSD__) || defined(__NetBSD__) struct arm_sync_icache_args arg; arg.addr = (uintptr_t)start; @@ -121,15 +122,13 @@ void __clear_cache(void *start, void *end) { : "=r"(start_reg) : "r"(syscall_nr), "r"(start_reg), "r"(end_reg), "r"(flags)); - if (start_reg != 0) { - compilerrt_abort(); - } + assert(start_reg == 0 && "Cache flush syscall failed."); #elif defined(_WIN32) FlushInstructionCache(GetCurrentProcess(), start, end - start); #else compilerrt_abort(); #endif -#elif defined(__mips__) && !defined(__FreeBSD__) +#elif defined(__linux__) && defined(__mips__) const uintptr_t start_int = (uintptr_t) start; const uintptr_t end_int = (uintptr_t) end; #if defined(__ANDROID__) && defined(__LP64__) @@ -165,6 +164,21 @@ void __clear_cache(void *start, void *end) { for (addr = xstart; addr < xend; addr += icache_line_size) __asm __volatile("ic ivau, %0" :: "r"(addr)); __asm __volatile("isb sy"); +#elif defined (__powerpc64__) + const size_t line_size = 32; + const size_t len = (uintptr_t)end - (uintptr_t)start; + + const uintptr_t mask = ~(line_size - 1); + const uintptr_t start_line = ((uintptr_t)start) & mask; + const uintptr_t end_line = ((uintptr_t)start + len + line_size - 1) & mask; + + for (uintptr_t line = start_line; line < end_line; line += line_size) + __asm__ volatile("dcbf 0, %0" : : "r"(line)); + __asm__ volatile("sync"); + + for (uintptr_t line = start_line; line < end_line; line += line_size) + __asm__ volatile("icbi 0, %0" : : "r"(line)); + __asm__ volatile("isync"); #else #if __APPLE__ /* On Darwin, sys_icache_invalidate() provides this functionality */ @@ -174,4 +188,3 @@ void __clear_cache(void *start, void *end) { #endif #endif } - diff --git a/contrib/compiler-rt/lib/builtins/comparedf2.c b/contrib/compiler-rt/lib/builtins/comparedf2.c index c5bb169d0021..44e5d2b288a6 100644 --- a/contrib/compiler-rt/lib/builtins/comparedf2.c +++ b/contrib/compiler-rt/lib/builtins/comparedf2.c @@ -143,8 +143,11 @@ __gtdf2(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) { return __unorddf2(a, b); } +#else +AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unorddf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/comparesf2.c b/contrib/compiler-rt/lib/builtins/comparesf2.c index 4badb5e1b9f7..43cd6a6a7003 100644 --- a/contrib/compiler-rt/lib/builtins/comparesf2.c +++ b/contrib/compiler-rt/lib/builtins/comparesf2.c @@ -143,8 +143,11 @@ __gtsf2(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { return __unordsf2(a, b); } +#else +AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unordsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/cpu_model.c b/contrib/compiler-rt/lib/builtins/cpu_model.c index 83ea7a49faf7..4c96e9cd85d5 100644 --- a/contrib/compiler-rt/lib/builtins/cpu_model.c +++ b/contrib/compiler-rt/lib/builtins/cpu_model.c @@ -54,6 +54,7 @@ enum ProcessorTypes { AMD_BTVER1, AMD_BTVER2, AMDFAM17H, + INTEL_KNM, CPU_TYPE_MAX }; @@ -74,6 +75,7 @@ enum ProcessorSubtypes { INTEL_COREI7_BROADWELL, INTEL_COREI7_SKYLAKE, INTEL_COREI7_SKYLAKE_AVX512, + INTEL_COREI7_CANNONLAKE, CPU_SUBTYPE_MAX }; @@ -339,6 +341,12 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" break; + // Cannonlake: + case 0x66: + *Type = INTEL_COREI7; + *Subtype = INTEL_COREI7_CANNONLAKE; // "cannonlake" + break; + case 0x1c: // Most 45 nm Intel Atom processors case 0x26: // 45 nm Atom Lincroft case 0x27: // 32 nm Atom Medfield @@ -361,6 +369,10 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, *Type = INTEL_KNL; // knl break; + case 0x85: + *Type = INTEL_KNM; // knm + break; + default: // Unknown family 6 CPU. break; break; diff --git a/contrib/compiler-rt/lib/builtins/divdf3.c b/contrib/compiler-rt/lib/builtins/divdf3.c index 492e32b851e9..04a4dc5571ca 100644 --- a/contrib/compiler-rt/lib/builtins/divdf3.c +++ b/contrib/compiler-rt/lib/builtins/divdf3.c @@ -183,8 +183,11 @@ __divdf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) { return __divdf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divdf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/divsf3.c b/contrib/compiler-rt/lib/builtins/divsf3.c index aa6289a6d70a..65294d70fc61 100644 --- a/contrib/compiler-rt/lib/builtins/divsf3.c +++ b/contrib/compiler-rt/lib/builtins/divsf3.c @@ -167,8 +167,11 @@ __divsf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) { return __divsf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/divsi3.c b/contrib/compiler-rt/lib/builtins/divsi3.c index 3852e3990b5b..75aea008ddc1 100644 --- a/contrib/compiler-rt/lib/builtins/divsi3.c +++ b/contrib/compiler-rt/lib/builtins/divsi3.c @@ -35,8 +35,5 @@ __divsi3(si_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) { - return __divsi3(a, b); -} +AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) COMPILER_RT_ALIAS(__divsi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/emutls.c b/contrib/compiler-rt/lib/builtins/emutls.c index 12aad3a42b76..5dd8dd154771 100644 --- a/contrib/compiler-rt/lib/builtins/emutls.c +++ b/contrib/compiler-rt/lib/builtins/emutls.c @@ -102,7 +102,6 @@ static __inline emutls_address_array* emutls_getspecific() { #include <malloc.h> #include <stdio.h> #include <assert.h> -#include <immintrin.h> static LPCRITICAL_SECTION emutls_mutex; static DWORD emutls_tls_index = TLS_OUT_OF_INDEXES; @@ -203,25 +202,24 @@ static __inline emutls_address_array* emutls_getspecific() { /* Provide atomic load/store functions for emutls_get_index if built with MSVC. */ #if !defined(__ATOMIC_RELEASE) +#include <intrin.h> enum { __ATOMIC_ACQUIRE = 2, __ATOMIC_RELEASE = 3 }; static __inline uintptr_t __atomic_load_n(void *ptr, unsigned type) { assert(type == __ATOMIC_ACQUIRE); + // These return the previous value - but since we do an OR with 0, + // it's equivalent to a plain load. #ifdef _WIN64 - return (uintptr_t) _load_be_u64(ptr); + return InterlockedOr64(ptr, 0); #else - return (uintptr_t) _load_be_u32(ptr); + return InterlockedOr(ptr, 0); #endif } static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) { assert(type == __ATOMIC_RELEASE); -#ifdef _WIN64 - _store_be_u64(ptr, val); -#else - _store_be_u32(ptr, val); -#endif + InterlockedExchangePointer((void *volatile *)ptr, (void *)val); } #endif diff --git a/contrib/compiler-rt/lib/builtins/enable_execute_stack.c b/contrib/compiler-rt/lib/builtins/enable_execute_stack.c index 0dc3482c4467..327d460b4253 100644 --- a/contrib/compiler-rt/lib/builtins/enable_execute_stack.c +++ b/contrib/compiler-rt/lib/builtins/enable_execute_stack.c @@ -22,7 +22,7 @@ #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN -#include <Windows.h> +#include <windows.h> #else #ifndef __APPLE__ #include <unistd.h> diff --git a/contrib/compiler-rt/lib/builtins/extendhfsf2.c b/contrib/compiler-rt/lib/builtins/extendhfsf2.c index e7d9fde8abfc..d9c0db84b0ce 100644 --- a/contrib/compiler-rt/lib/builtins/extendhfsf2.c +++ b/contrib/compiler-rt/lib/builtins/extendhfsf2.c @@ -23,8 +23,11 @@ COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_h2f(uint16_t a) { return __extendhfsf2(a); } +#else +AEABI_RTABI float __aeabi_h2f(uint16_t a) COMPILER_RT_ALIAS(__extendhfsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/extendsfdf2.c b/contrib/compiler-rt/lib/builtins/extendsfdf2.c index b9e7a7471a98..3d84529a6c53 100644 --- a/contrib/compiler-rt/lib/builtins/extendsfdf2.c +++ b/contrib/compiler-rt/lib/builtins/extendsfdf2.c @@ -17,8 +17,11 @@ COMPILER_RT_ABI double __extendsfdf2(float a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_f2d(float a) { return __extendsfdf2(a); } +#else +AEABI_RTABI double __aeabi_f2d(float a) COMPILER_RT_ALIAS(__extendsfdf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixdfdi.c b/contrib/compiler-rt/lib/builtins/fixdfdi.c index 31d76df28255..54e312d3c8f7 100644 --- a/contrib/compiler-rt/lib/builtins/fixdfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixdfdi.c @@ -45,13 +45,11 @@ __fixdfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI di_int -#if defined(__SOFT_FP__) -__aeabi_d2lz(fp_t a) { -#else -__aeabi_d2lz(double a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI di_int __aeabi_d2lz(fp_t a) { return __fixdfdi(a); } +#else +AEABI_RTABI di_int __aeabi_d2lz(fp_t a) COMPILER_RT_ALIAS(__fixdfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixdfsi.c b/contrib/compiler-rt/lib/builtins/fixdfsi.c index fc316dcd0545..5b9588175717 100644 --- a/contrib/compiler-rt/lib/builtins/fixdfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixdfsi.c @@ -20,8 +20,11 @@ __fixdfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI si_int __aeabi_d2iz(fp_t a) { return __fixdfsi(a); } +#else +AEABI_RTABI si_int __aeabi_d2iz(fp_t a) COMPILER_RT_ALIAS(__fixdfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixsfdi.c b/contrib/compiler-rt/lib/builtins/fixsfdi.c index c43473637d60..32e87c60889f 100644 --- a/contrib/compiler-rt/lib/builtins/fixsfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixsfdi.c @@ -45,13 +45,11 @@ __fixsfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI di_int -#if defined(__SOFT_FP__) -__aeabi_f2lz(fp_t a) { -#else -__aeabi_f2lz(float a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI di_int __aeabi_f2lz(fp_t a) { return __fixsfdi(a); } +#else +AEABI_RTABI di_int __aeabi_f2lz(fp_t a) COMPILER_RT_ALIAS(__fixsfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixsfsi.c b/contrib/compiler-rt/lib/builtins/fixsfsi.c index 3276df966460..e94e5f3dcd68 100644 --- a/contrib/compiler-rt/lib/builtins/fixsfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixsfsi.c @@ -20,8 +20,11 @@ __fixsfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI si_int __aeabi_f2iz(fp_t a) { return __fixsfsi(a); } +#else +AEABI_RTABI si_int __aeabi_f2iz(fp_t a) COMPILER_RT_ALIAS(__fixsfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunsdfdi.c b/contrib/compiler-rt/lib/builtins/fixunsdfdi.c index b734409709bf..bfe4dbb25656 100644 --- a/contrib/compiler-rt/lib/builtins/fixunsdfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixunsdfdi.c @@ -42,13 +42,11 @@ __fixunsdfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI du_int -#if defined(__SOFT_FP__) -__aeabi_d2ulz(fp_t a) { -#else -__aeabi_d2ulz(double a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) { return __fixunsdfdi(a); } +#else +AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunsdfsi.c b/contrib/compiler-rt/lib/builtins/fixunsdfsi.c index bb3d8e0f831b..3c5355beae1a 100644 --- a/contrib/compiler-rt/lib/builtins/fixunsdfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixunsdfsi.c @@ -19,8 +19,11 @@ __fixunsdfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) { return __fixunsdfsi(a); } +#else +AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunssfdi.c b/contrib/compiler-rt/lib/builtins/fixunssfdi.c index 5d92245df0d9..080a25bb1e99 100644 --- a/contrib/compiler-rt/lib/builtins/fixunssfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixunssfdi.c @@ -43,13 +43,11 @@ __fixunssfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI du_int -#if defined(__SOFT_FP__) -__aeabi_f2ulz(fp_t a) { -#else -__aeabi_f2ulz(float a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) { return __fixunssfdi(a); } +#else +AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunssfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunssfsi.c b/contrib/compiler-rt/lib/builtins/fixunssfsi.c index 91d5e8ae5d7f..eca2916a5c88 100644 --- a/contrib/compiler-rt/lib/builtins/fixunssfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixunssfsi.c @@ -23,8 +23,11 @@ __fixunssfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) { return __fixunssfsi(a); } +#else +AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunssfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatdidf.c b/contrib/compiler-rt/lib/builtins/floatdidf.c index 681fecef9682..36b856e078d4 100644 --- a/contrib/compiler-rt/lib/builtins/floatdidf.c +++ b/contrib/compiler-rt/lib/builtins/floatdidf.c @@ -105,8 +105,11 @@ __floatdidf(di_int a) #endif #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_l2d(di_int a) { return __floatdidf(a); } +#else +AEABI_RTABI double __aeabi_l2d(di_int a) COMPILER_RT_ALIAS(__floatdidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatdisf.c b/contrib/compiler-rt/lib/builtins/floatdisf.c index dd548165c373..a2f09eb2ed2c 100644 --- a/contrib/compiler-rt/lib/builtins/floatdisf.c +++ b/contrib/compiler-rt/lib/builtins/floatdisf.c @@ -78,8 +78,11 @@ __floatdisf(di_int a) } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_l2f(di_int a) { return __floatdisf(a); } +#else +AEABI_RTABI float __aeabi_l2f(di_int a) COMPILER_RT_ALIAS(__floatdisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatsidf.c b/contrib/compiler-rt/lib/builtins/floatsidf.c index 2ae395bdc1db..fe051123ce7c 100644 --- a/contrib/compiler-rt/lib/builtins/floatsidf.c +++ b/contrib/compiler-rt/lib/builtins/floatsidf.c @@ -51,8 +51,11 @@ __floatsidf(int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_i2d(int a) { return __floatsidf(a); } +#else +AEABI_RTABI fp_t __aeabi_i2d(int a) COMPILER_RT_ALIAS(__floatsidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatsisf.c b/contrib/compiler-rt/lib/builtins/floatsisf.c index 08891fcdf201..bf087ee3c295 100644 --- a/contrib/compiler-rt/lib/builtins/floatsisf.c +++ b/contrib/compiler-rt/lib/builtins/floatsisf.c @@ -57,8 +57,11 @@ __floatsisf(int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_i2f(int a) { return __floatsisf(a); } +#else +AEABI_RTABI fp_t __aeabi_i2f(int a) COMPILER_RT_ALIAS(__floatsisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatundidf.c b/contrib/compiler-rt/lib/builtins/floatundidf.c index 6c1a931ef2f3..8bc2a096324f 100644 --- a/contrib/compiler-rt/lib/builtins/floatundidf.c +++ b/contrib/compiler-rt/lib/builtins/floatundidf.c @@ -104,8 +104,11 @@ __floatundidf(du_int a) #endif #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_ul2d(du_int a) { return __floatundidf(a); } +#else +AEABI_RTABI double __aeabi_ul2d(du_int a) COMPILER_RT_ALIAS(__floatundidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatundisf.c b/contrib/compiler-rt/lib/builtins/floatundisf.c index 86841a75dc66..844786ea7777 100644 --- a/contrib/compiler-rt/lib/builtins/floatundisf.c +++ b/contrib/compiler-rt/lib/builtins/floatundisf.c @@ -75,8 +75,11 @@ __floatundisf(du_int a) } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_ul2f(du_int a) { return __floatundisf(a); } +#else +AEABI_RTABI float __aeabi_ul2f(du_int a) COMPILER_RT_ALIAS(__floatundisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatunsidf.c b/contrib/compiler-rt/lib/builtins/floatunsidf.c index 8d4807194f0b..75cf6b9177df 100644 --- a/contrib/compiler-rt/lib/builtins/floatunsidf.c +++ b/contrib/compiler-rt/lib/builtins/floatunsidf.c @@ -40,8 +40,11 @@ __floatunsidf(unsigned int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) { return __floatunsidf(a); } +#else +AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) COMPILER_RT_ALIAS(__floatunsidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatunsisf.c b/contrib/compiler-rt/lib/builtins/floatunsisf.c index f194c046d2fb..29525ccedbbe 100644 --- a/contrib/compiler-rt/lib/builtins/floatunsisf.c +++ b/contrib/compiler-rt/lib/builtins/floatunsisf.c @@ -48,8 +48,11 @@ __floatunsisf(unsigned int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) { return __floatunsisf(a); } +#else +AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) COMPILER_RT_ALIAS(__floatunsisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/int_endianness.h b/contrib/compiler-rt/lib/builtins/int_endianness.h index 7995ddbb953c..e2586c56bac8 100644 --- a/contrib/compiler-rt/lib/builtins/int_endianness.h +++ b/contrib/compiler-rt/lib/builtins/int_endianness.h @@ -61,7 +61,7 @@ #endif /* *BSD */ -#if defined(__OpenBSD__) || defined(__Bitrig__) +#if defined(__OpenBSD__) #include <machine/endian.h> #if _BYTE_ORDER == _BIG_ENDIAN @@ -72,7 +72,7 @@ #define _YUGA_BIG_ENDIAN 0 #endif /* _BYTE_ORDER */ -#endif /* OpenBSD and Bitrig. */ +#endif /* OpenBSD */ /* .. */ diff --git a/contrib/compiler-rt/lib/builtins/int_lib.h b/contrib/compiler-rt/lib/builtins/int_lib.h index 53caf3a53e63..93394cebf7ea 100644 --- a/contrib/compiler-rt/lib/builtins/int_lib.h +++ b/contrib/compiler-rt/lib/builtins/int_lib.h @@ -22,9 +22,11 @@ #if defined(__ELF__) #define FNALIAS(alias_name, original_name) \ - void alias_name() __attribute__((alias(#original_name))) + void alias_name() __attribute__((__alias__(#original_name))) +#define COMPILER_RT_ALIAS(aliasee) __attribute__((__alias__(#aliasee))) #else #define FNALIAS(alias, name) _Pragma("GCC error(\"alias unsupported on this file format\")") +#define COMPILER_RT_ALIAS(aliasee) _Pragma("GCC error(\"alias unsupported on this file format\")") #endif /* ABI macro definitions */ diff --git a/contrib/compiler-rt/lib/builtins/lshrdi3.c b/contrib/compiler-rt/lib/builtins/lshrdi3.c index becbbef4eb09..67b2a7668345 100644 --- a/contrib/compiler-rt/lib/builtins/lshrdi3.c +++ b/contrib/compiler-rt/lib/builtins/lshrdi3.c @@ -41,8 +41,5 @@ __lshrdi3(di_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) { - return __lshrdi3(a, b); -} +AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) COMPILER_RT_ALIAS(__lshrdi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/muldf3.c b/contrib/compiler-rt/lib/builtins/muldf3.c index 59a60190eba3..1bb103e38c13 100644 --- a/contrib/compiler-rt/lib/builtins/muldf3.c +++ b/contrib/compiler-rt/lib/builtins/muldf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI fp_t __muldf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) { return __muldf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__muldf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/muldi3.c b/contrib/compiler-rt/lib/builtins/muldi3.c index 6818a9e2f722..a187315e9165 100644 --- a/contrib/compiler-rt/lib/builtins/muldi3.c +++ b/contrib/compiler-rt/lib/builtins/muldi3.c @@ -54,8 +54,5 @@ __muldi3(di_int a, di_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) { - return __muldi3(a, b); -} +AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) COMPILER_RT_ALIAS(__muldi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/mulsf3.c b/contrib/compiler-rt/lib/builtins/mulsf3.c index f141af1acc58..1e2cf3e717c9 100644 --- a/contrib/compiler-rt/lib/builtins/mulsf3.c +++ b/contrib/compiler-rt/lib/builtins/mulsf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI fp_t __mulsf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) { return __mulsf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__mulsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/negdf2.c b/contrib/compiler-rt/lib/builtins/negdf2.c index 5e2544cdb4be..f0bfaad24743 100644 --- a/contrib/compiler-rt/lib/builtins/negdf2.c +++ b/contrib/compiler-rt/lib/builtins/negdf2.c @@ -20,8 +20,11 @@ __negdf2(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_dneg(fp_t a) { return __negdf2(a); } +#else +AEABI_RTABI fp_t __aeabi_dneg(fp_t a) COMPILER_RT_ALIAS(__negdf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/negsf2.c b/contrib/compiler-rt/lib/builtins/negsf2.c index f90b34335680..05c97d4d5a11 100644 --- a/contrib/compiler-rt/lib/builtins/negsf2.c +++ b/contrib/compiler-rt/lib/builtins/negsf2.c @@ -20,8 +20,11 @@ __negsf2(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fneg(fp_t a) { return __negsf2(a); } +#else +AEABI_RTABI fp_t __aeabi_fneg(fp_t a) COMPILER_RT_ALIAS(__negsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/subdf3.c b/contrib/compiler-rt/lib/builtins/subdf3.c index 38340dfab1a6..a892fa603cf2 100644 --- a/contrib/compiler-rt/lib/builtins/subdf3.c +++ b/contrib/compiler-rt/lib/builtins/subdf3.c @@ -22,8 +22,11 @@ __subdf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) { return __subdf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subdf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/subsf3.c b/contrib/compiler-rt/lib/builtins/subsf3.c index 34276b1447ba..4b2786177dcc 100644 --- a/contrib/compiler-rt/lib/builtins/subsf3.c +++ b/contrib/compiler-rt/lib/builtins/subsf3.c @@ -22,8 +22,11 @@ __subsf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) { return __subsf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/truncdfhf2.c b/contrib/compiler-rt/lib/builtins/truncdfhf2.c index 4bb71aa178a0..8354a41b8b6f 100644 --- a/contrib/compiler-rt/lib/builtins/truncdfhf2.c +++ b/contrib/compiler-rt/lib/builtins/truncdfhf2.c @@ -16,8 +16,11 @@ COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI uint16_t __aeabi_d2h(double a) { return __truncdfhf2(a); } +#else +AEABI_RTABI uint16_t __aeabi_d2h(double a) COMPILER_RT_ALIAS(__truncdfhf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/truncdfsf2.c b/contrib/compiler-rt/lib/builtins/truncdfsf2.c index 8bf58bb23a3b..195d3e0656e7 100644 --- a/contrib/compiler-rt/lib/builtins/truncdfsf2.c +++ b/contrib/compiler-rt/lib/builtins/truncdfsf2.c @@ -16,8 +16,11 @@ COMPILER_RT_ABI float __truncdfsf2(double a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_d2f(double a) { return __truncdfsf2(a); } +#else +AEABI_RTABI float __aeabi_d2f(double a) COMPILER_RT_ALIAS(__truncdfsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/truncsfhf2.c b/contrib/compiler-rt/lib/builtins/truncsfhf2.c index f6ce1fa1de05..9c84ab4f938a 100644 --- a/contrib/compiler-rt/lib/builtins/truncsfhf2.c +++ b/contrib/compiler-rt/lib/builtins/truncsfhf2.c @@ -22,8 +22,11 @@ COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI uint16_t __aeabi_f2h(float a) { return __truncsfhf2(a); } +#else +AEABI_RTABI uint16_t __aeabi_f2h(float a) COMPILER_RT_ALIAS(__truncsfhf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/udivsi3.c b/contrib/compiler-rt/lib/builtins/udivsi3.c index 8eccf102cc97..bb720f8c382b 100644 --- a/contrib/compiler-rt/lib/builtins/udivsi3.c +++ b/contrib/compiler-rt/lib/builtins/udivsi3.c @@ -64,8 +64,5 @@ __udivsi3(su_int n, su_int d) } #if defined(__ARM_EABI__) -AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) { - return __udivsi3(n, d); -} +AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) COMPILER_RT_ALIAS(__udivsi3); #endif - diff --git a/contrib/compiler-rt/lib/cfi/cfi.cc b/contrib/compiler-rt/lib/cfi/cfi.cc index f720230a70be..f7693b37d29c 100644 --- a/contrib/compiler-rt/lib/cfi/cfi.cc +++ b/contrib/compiler-rt/lib/cfi/cfi.cc @@ -280,7 +280,7 @@ void InitShadow() { CHECK_EQ(0, GetShadow()); CHECK_EQ(0, GetShadowSize()); - uptr vma = GetMaxVirtualAddress(); + uptr vma = GetMaxUserVirtualAddress(); // Shadow is 2 -> 2**kShadowGranularity. SetShadowSize((vma >> (kShadowGranularity - 1)) + 1); VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize()); diff --git a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt index cc111be8120e..d8c9d49e3cdd 100644 --- a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt +++ b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt @@ -1,18 +1,4 @@ -# Standard library types. -type:std::* - -# The stdext namespace contains Microsoft standard library extensions. -type:stdext::* - -# Types with a uuid attribute, i.e. COM types. -type:attr:uuid - -# STL allocators (T *allocator<T *>::allocate(size_type, const void*)). -# The type signature mandates a cast from uninitialized void* to T*. -# size_type can either be unsigned int (j) or unsigned long (m). -fun:*8allocateEjPKv -fun:*8allocateEmPKv - +[cfi-unrelated-cast] # std::get_temporary_buffer, likewise (libstdc++, libc++). fun:_ZSt20get_temporary_buffer* fun:_ZNSt3__120get_temporary_buffer* diff --git a/contrib/compiler-rt/lib/dfsan/dfsan.cc b/contrib/compiler-rt/lib/dfsan/dfsan.cc index 3aa99b7f9c27..9e360c959fa7 100644 --- a/contrib/compiler-rt/lib/dfsan/dfsan.cc +++ b/contrib/compiler-rt/lib/dfsan/dfsan.cc @@ -21,6 +21,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_libc.h" diff --git a/contrib/compiler-rt/lib/esan/esan_sideline.h b/contrib/compiler-rt/lib/esan/esan_sideline.h index aa3fae1db318..04aff22f4cf0 100644 --- a/contrib/compiler-rt/lib/esan/esan_sideline.h +++ b/contrib/compiler-rt/lib/esan/esan_sideline.h @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" namespace __esan { @@ -46,7 +47,8 @@ public: private: static int runSideline(void *Arg); static void registerSignal(int SigNum); - static void handleSidelineSignal(int SigNum, void *SigInfo, void *Ctx); + static void handleSidelineSignal(int SigNum, __sanitizer_siginfo *SigInfo, + void *Ctx); char *Stack; SidelineFunc sampleFunc; diff --git a/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp b/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp index bc272dfe49f8..4a96910ece35 100644 --- a/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp +++ b/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp @@ -40,7 +40,8 @@ static const uptr SidelineIdUninitialized = 1; static SidelineThread *TheThread; // We aren't passing SA_NODEFER so the same signal is blocked while here. -void SidelineThread::handleSidelineSignal(int SigNum, void *SigInfo, +void SidelineThread::handleSidelineSignal(int SigNum, + __sanitizer_siginfo *SigInfo, void *Ctx) { VPrintf(3, "Sideline signal %d\n", SigNum); CHECK_EQ(SigNum, SIGALRM); diff --git a/contrib/compiler-rt/lib/esan/working_set_posix.cpp b/contrib/compiler-rt/lib/esan/working_set_posix.cpp index fcfa87128857..5ec53b959261 100644 --- a/contrib/compiler-rt/lib/esan/working_set_posix.cpp +++ b/contrib/compiler-rt/lib/esan/working_set_posix.cpp @@ -34,7 +34,7 @@ bool processWorkingSetSignal(int SigNum, void (*Handler)(int), VPrintf(2, "%s: %d\n", __FUNCTION__, SigNum); if (SigNum == SIGSEGV) { *Result = AppSigAct.handler; - AppSigAct.sigaction = (void (*)(int, void*, void*))Handler; + AppSigAct.sigaction = (decltype(AppSigAct.sigaction))Handler; return false; // Skip real call. } return true; @@ -73,7 +73,7 @@ bool processWorkingSetSigprocmask(int How, void *Set, void *OldSet) { static void reinstateDefaultHandler(int SigNum) { __sanitizer_sigaction SigAct; internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = (void (*)(int, void*, void*)) SIG_DFL; + SigAct.sigaction = (decltype(SigAct.sigaction))SIG_DFL; int Res = internal_sigaction(SigNum, &SigAct, nullptr); CHECK(Res == 0); VPrintf(1, "Unregistered for %d handler\n", SigNum); @@ -81,7 +81,8 @@ static void reinstateDefaultHandler(int SigNum) { // If this is a shadow fault, we handle it here; otherwise, we pass it to the // app to handle it just as the app would do without our tool in place. -static void handleMemoryFault(int SigNum, void *Info, void *Ctx) { +static void handleMemoryFault(int SigNum, __sanitizer_siginfo *Info, + void *Ctx) { if (SigNum == SIGSEGV) { // We rely on si_addr being filled in (thus we do not support old kernels). siginfo_t *SigInfo = (siginfo_t *)Info; diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.cc b/contrib/compiler-rt/lib/hwasan/hwasan.cc new file mode 100644 index 000000000000..fcc40eb90182 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan.cc @@ -0,0 +1,303 @@ +//===-- hwasan.cc -----------------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// HWAddressSanitizer runtime. +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "ubsan/ubsan_flags.h" +#include "ubsan/ubsan_init.h" + +// ACHTUNG! No system header includes in this file. + +using namespace __sanitizer; + +namespace __hwasan { + +void EnterSymbolizer() { + HwasanThread *t = GetCurrentThread(); + CHECK(t); + t->EnterSymbolizer(); +} +void ExitSymbolizer() { + HwasanThread *t = GetCurrentThread(); + CHECK(t); + t->LeaveSymbolizer(); +} +bool IsInSymbolizer() { + HwasanThread *t = GetCurrentThread(); + return t && t->InSymbolizer(); +} + +static Flags hwasan_flags; + +Flags *flags() { + return &hwasan_flags; +} + +int hwasan_inited = 0; +bool hwasan_init_is_running; + +int hwasan_report_count = 0; + +void Flags::SetDefaults() { +#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "hwasan_flags.inc" +#undef HWASAN_FLAG +} + +static void RegisterHwasanFlags(FlagParser *parser, Flags *f) { +#define HWASAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "hwasan_flags.inc" +#undef HWASAN_FLAG +} + +static void InitializeFlags() { + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH"); + cf.malloc_context_size = 20; + cf.handle_ioctl = true; + // FIXME: test and enable. + cf.check_printf = false; + cf.intercept_tls_get_addr = true; + cf.exitcode = 99; + cf.handle_sigill = kHandleSignalExclusive; + OverrideCommonFlags(cf); + } + + Flags *f = flags(); + f->SetDefaults(); + + FlagParser parser; + RegisterHwasanFlags(&parser, f); + RegisterCommonFlags(&parser); + +#if HWASAN_CONTAINS_UBSAN + __ubsan::Flags *uf = __ubsan::flags(); + uf->SetDefaults(); + + FlagParser ubsan_parser; + __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); + RegisterCommonFlags(&ubsan_parser); +#endif + + // Override from user-specified string. + if (__hwasan_default_options) + parser.ParseString(__hwasan_default_options()); +#if HWASAN_CONTAINS_UBSAN + const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); + ubsan_parser.ParseString(ubsan_default_options); +#endif + + const char *hwasan_options = GetEnv("HWASAN_OPTIONS"); + parser.ParseString(hwasan_options); +#if HWASAN_CONTAINS_UBSAN + ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); +#endif + VPrintf(1, "HWASAN_OPTIONS: %s\n", hwasan_options ? hwasan_options : "<empty>"); + + InitializeCommonFlags(); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) parser.PrintFlagDescriptions(); +} + +void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, + void *context, bool request_fast_unwind) { + HwasanThread *t = GetCurrentThread(); + if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); + } + stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), + request_fast_unwind); +} + +void PrintWarning(uptr pc, uptr bp) { + GET_FATAL_STACK_TRACE_PC_BP(pc, bp); + ReportInvalidAccess(&stack, 0); +} + +} // namespace __hwasan + +// Interface. + +using namespace __hwasan; + +void __hwasan_init() { + CHECK(!hwasan_init_is_running); + if (hwasan_inited) return; + hwasan_init_is_running = 1; + SanitizerToolName = "HWAddressSanitizer"; + + InitTlsSize(); + + CacheBinaryName(); + InitializeFlags(); + + __sanitizer_set_report_path(common_flags()->log_path); + + InitializeInterceptors(); + InstallDeadlySignalHandlers(HwasanOnDeadlySignal); + InstallAtExitHandler(); // Needs __cxa_atexit interceptor. + + DisableCoreDumperIfNecessary(); + if (!InitShadow()) { + Printf("FATAL: HWAddressSanitizer can not mmap the shadow memory.\n"); + Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); + Printf("FATAL: Disabling ASLR is known to cause this error.\n"); + Printf("FATAL: If running under GDB, try " + "'set disable-randomization off'.\n"); + DumpProcessMap(); + Die(); + } + + Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); + + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); + + HwasanTSDInit(HwasanTSDDtor); + + HwasanAllocatorInit(); + + HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr); + SetCurrentThread(main_thread); + main_thread->ThreadStart(); + +#if HWASAN_CONTAINS_UBSAN + __ubsan::InitAsPlugin(); +#endif + + VPrintf(1, "HWAddressSanitizer init done\n"); + + hwasan_init_is_running = 0; + hwasan_inited = 1; +} + +void __hwasan_print_shadow(const void *x, uptr size) { + // FIXME: + Printf("FIXME: __hwasan_print_shadow unimplemented\n"); +} + +sptr __hwasan_test_shadow(const void *p, uptr sz) { + if (sz == 0) + return -1; + tag_t ptr_tag = GetTagFromPointer((uptr)p); + if (ptr_tag == 0) + return -1; + uptr ptr_raw = GetAddressFromPointer((uptr)p); + uptr shadow_first = MEM_TO_SHADOW(ptr_raw); + uptr shadow_last = MEM_TO_SHADOW(ptr_raw + sz - 1); + for (uptr s = shadow_first; s <= shadow_last; ++s) + if (*(tag_t*)s != ptr_tag) + return SHADOW_TO_MEM(s) - ptr_raw; + return -1; +} + +u16 __sanitizer_unaligned_load16(const uu16 *p) { + return *p; +} +u32 __sanitizer_unaligned_load32(const uu32 *p) { + return *p; +} +u64 __sanitizer_unaligned_load64(const uu64 *p) { + return *p; +} +void __sanitizer_unaligned_store16(uu16 *p, u16 x) { + *p = x; +} +void __sanitizer_unaligned_store32(uu32 *p, u32 x) { + *p = x; +} +void __sanitizer_unaligned_store64(uu64 *p, u64 x) { + *p = x; +} + +template<unsigned X> +__attribute__((always_inline)) +static void SigIll() { +#if defined(__aarch64__) + asm("hlt %0\n\t" ::"n"(X)); +#elif defined(__x86_64__) || defined(__i386__) + asm("ud2\n\t"); +#else + // FIXME: not always sigill. + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + +template<bool IsStore, unsigned LogSize> +__attribute__((always_inline, nodebug)) +static void CheckAddress(uptr p) { + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw); + if (UNLIKELY(ptr_tag != mem_tag)) SigIll<0x100 + 0x10 * IsStore + LogSize>(); +} + +template<bool IsStore> +__attribute__((always_inline, nodebug)) +static void CheckAddressSized(uptr p, uptr sz) { + CHECK_NE(0, sz); + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw); + tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1); + for (tag_t *t = shadow_first; t <= shadow_last; ++t) + if (UNLIKELY(ptr_tag != *t)) SigIll<0x100 + 0x10 * IsStore + 0xf>(); +} + +void __hwasan_load(uptr p, uptr sz) { CheckAddressSized<false>(p, sz); } +void __hwasan_load1(uptr p) { CheckAddress<false, 0>(p); } +void __hwasan_load2(uptr p) { CheckAddress<false, 1>(p); } +void __hwasan_load4(uptr p) { CheckAddress<false, 2>(p); } +void __hwasan_load8(uptr p) { CheckAddress<false, 3>(p); } +void __hwasan_load16(uptr p) { CheckAddress<false, 4>(p); } + +void __hwasan_store(uptr p, uptr sz) { CheckAddressSized<true>(p, sz); } +void __hwasan_store1(uptr p) { CheckAddress<true, 0>(p); } +void __hwasan_store2(uptr p) { CheckAddress<true, 1>(p); } +void __hwasan_store4(uptr p) { CheckAddress<true, 2>(p); } +void __hwasan_store8(uptr p) { CheckAddress<true, 3>(p); } +void __hwasan_store16(uptr p) { CheckAddress<true, 4>(p); } + +#if !SANITIZER_SUPPORTS_WEAK_HOOKS +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char* __hwasan_default_options() { return ""; } +} // extern "C" +#endif + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_print_stack_trace() { + GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); + stack.Print(); +} +} // extern "C" diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.h b/contrib/compiler-rt/lib/hwasan/hwasan.h new file mode 100644 index 000000000000..bcf5282dce7c --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan.h @@ -0,0 +1,172 @@ +//===-- hwasan.h --------------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Private Hwasan header. +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_H +#define HWASAN_H + +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "hwasan_interface_internal.h" +#include "hwasan_flags.h" +#include "ubsan/ubsan_platform.h" + +#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE +# define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE 1 +#endif + +#ifndef HWASAN_CONTAINS_UBSAN +# define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB +#endif + +typedef u8 tag_t; + +// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th). +const uptr kShadowScale = 4; +const uptr kShadowAlignment = 1UL << kShadowScale; + +#define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale) +#define MEM_TO_SHADOW(mem) ((uptr)(mem) >> kShadowScale) +#define SHADOW_TO_MEM(shadow) ((uptr)(shadow) << kShadowScale) + +#define MEM_IS_APP(mem) true + +// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address +// translation and can be used to store a tag. +const unsigned kAddressTagShift = 56; +const uptr kAddressTagMask = 0xFFUL << kAddressTagShift; + +static inline tag_t GetTagFromPointer(uptr p) { + return p >> kAddressTagShift; +} + +static inline uptr GetAddressFromPointer(uptr p) { + return p & ~kAddressTagMask; +} + +static inline void * GetAddressFromPointer(const void *p) { + return (void *)((uptr)p & ~kAddressTagMask); +} + +static inline uptr AddTagToPointer(uptr p, tag_t tag) { + return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift); +} + +namespace __hwasan { + +extern int hwasan_inited; +extern bool hwasan_init_is_running; +extern int hwasan_report_count; + +bool ProtectRange(uptr beg, uptr end); +bool InitShadow(); +char *GetProcSelfMaps(); +void InitializeInterceptors(); + +void HwasanAllocatorInit(); +void HwasanAllocatorThreadFinish(); +void HwasanDeallocate(StackTrace *stack, void *ptr); + +void *hwasan_malloc(uptr size, StackTrace *stack); +void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); +void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack); +void *hwasan_valloc(uptr size, StackTrace *stack); +void *hwasan_pvalloc(uptr size, StackTrace *stack); +void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); +void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack); +int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, + StackTrace *stack); + +void InstallTrapHandler(); +void InstallAtExitHandler(); + +const char *GetStackOriginDescr(u32 id, uptr *pc); + +void EnterSymbolizer(); +void ExitSymbolizer(); +bool IsInSymbolizer(); + +struct SymbolizerScope { + SymbolizerScope() { EnterSymbolizer(); } + ~SymbolizerScope() { ExitSymbolizer(); } +}; + +void PrintWarning(uptr pc, uptr bp); + +void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, + void *context, bool request_fast_unwind); + +void ReportInvalidAccess(StackTrace *stack, u32 origin); +void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, + bool is_store); +void ReportStats(); +void ReportAtExitStatistics(); +void DescribeMemoryRange(const void *x, uptr size); +void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, uptr size, + uptr offset); + +// Returns a "chained" origin id, pointing to the given stack trace followed by +// the previous origin id. +u32 ChainOrigin(u32 id, StackTrace *stack); + +const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; + +#define GET_MALLOC_STACK_TRACE \ + BufferedStackTrace stack; \ + if (hwasan_inited) \ + GetStackTrace(&stack, common_flags()->malloc_context_size, \ + StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ + common_flags()->fast_unwind_on_malloc) + +#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (hwasan_inited) \ + GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ + common_flags()->fast_unwind_on_fatal) + +class ScopedThreadLocalStateBackup { + public: + ScopedThreadLocalStateBackup() { Backup(); } + ~ScopedThreadLocalStateBackup() { Restore(); } + void Backup(); + void Restore(); + private: + u64 va_arg_overflow_size_tls; +}; + +void HwasanTSDInit(void (*destructor)(void *tsd)); +void *HwasanTSDGet(); +void HwasanTSDSet(void *tsd); +void HwasanTSDDtor(void *tsd); + +void HwasanOnDeadlySignal(int signo, void *info, void *context); + +} // namespace __hwasan + +#define HWASAN_MALLOC_HOOK(ptr, size) \ + do { \ + if (&__sanitizer_malloc_hook) { \ + __sanitizer_malloc_hook(ptr, size); \ + } \ + RunMallocHooks(ptr, size); \ + } while (false) +#define HWASAN_FREE_HOOK(ptr) \ + do { \ + if (&__sanitizer_free_hook) { \ + __sanitizer_free_hook(ptr); \ + } \ + RunFreeHooks(ptr); \ + } while (false) + +#endif // HWASAN_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra b/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra new file mode 100644 index 000000000000..8738627e06e8 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra @@ -0,0 +1,2 @@ +__hwasan_* +__ubsan_* diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc new file mode 100644 index 000000000000..fbcaf78b88f0 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc @@ -0,0 +1,330 @@ +//===-- hwasan_allocator.cc --------------------------- ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// HWAddressSanitizer allocator. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "hwasan.h" +#include "hwasan_allocator.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" + +namespace __hwasan { + +enum { + CHUNK_INVALID = 0, + CHUNK_FREE = 1, + CHUNK_ALLOCATED = 2 +}; + +struct Metadata { + u64 state : 2; + u64 requested_size : 62; + u32 alloc_context_id; + u32 free_context_id; +}; + +bool HwasanChunkView::IsValid() const { + return metadata_ && metadata_->state != CHUNK_INVALID; +} +bool HwasanChunkView::IsAllocated() const { + return metadata_ && metadata_->state == CHUNK_ALLOCATED; +} +uptr HwasanChunkView::Beg() const { + return block_; +} +uptr HwasanChunkView::End() const { + return Beg() + UsedSize(); +} +uptr HwasanChunkView::UsedSize() const { + return metadata_->requested_size; +} +u32 HwasanChunkView::GetAllocStackId() const { + return metadata_->alloc_context_id; +} +u32 HwasanChunkView::GetFreeStackId() const { + return metadata_->free_context_id; +} + +struct HwasanMapUnmapCallback { + void OnMap(uptr p, uptr size) const {} + void OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // It can return as user-requested mmap() or another thread stack. + // Make it accessible with zero-tagged pointer. + TagMemory(p, size, 0); + } +}; + +#if !defined(__aarch64__) +#error unsupported platform +#endif + +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G +static const uptr kRegionSizeLog = 20; +static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; +typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog; + typedef __hwasan::ByteMap ByteMap; + typedef HwasanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32<AP32> PrimaryAllocator; +typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; +typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator; +typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, + SecondaryAllocator> Allocator; + +static Allocator allocator; +static AllocatorCache fallback_allocator_cache; +static SpinMutex fallback_mutex; +static atomic_uint8_t hwasan_allocator_tagging_enabled; + +void HwasanAllocatorInit() { + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, + !flags()->disable_allocator_tagging); + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + allocator.Init(common_flags()->allocator_release_to_os_interval_ms); +} + +AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) { + CHECK(ms); + CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); + return reinterpret_cast<AllocatorCache *>(ms->allocator_cache); +} + +void HwasanThreadLocalMallocStorage::CommitBack() { + allocator.SwallowCache(GetAllocatorCache(this)); +} + +static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, + bool zeroise) { + alignment = Max(alignment, kShadowAlignment); + size = RoundUpTo(size, kShadowAlignment); + + if (size > kMaxAllowedMallocSize) { + Report("WARNING: HWAddressSanitizer failed to allocate %p bytes\n", + (void *)size); + return Allocator::FailureHandler::OnBadRequest(); + } + HwasanThread *t = GetCurrentThread(); + void *allocated; + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocated = allocator.Allocate(cache, size, alignment); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocated = allocator.Allocate(cache, size, alignment); + } + Metadata *meta = + reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated)); + meta->state = CHUNK_ALLOCATED; + meta->requested_size = size; + meta->alloc_context_id = StackDepotPut(*stack); + if (zeroise) + internal_memset(allocated, 0, size); + + void *user_ptr = (flags()->tag_in_malloc && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) + ? (void *)TagMemoryAligned((uptr)allocated, size, 0xBB) + : allocated; + + HWASAN_MALLOC_HOOK(user_ptr, size); + return user_ptr; +} + +void HwasanDeallocate(StackTrace *stack, void *user_ptr) { + CHECK(user_ptr); + HWASAN_FREE_HOOK(user_ptr); + + void *p = GetAddressFromPointer(user_ptr); + Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p)); + uptr size = meta->requested_size; + meta->state = CHUNK_FREE; + meta->requested_size = 0; + meta->free_context_id = StackDepotPut(*stack); + // This memory will not be reused by anyone else, so we are free to keep it + // poisoned. + if (flags()->tag_in_free && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) + TagMemoryAligned((uptr)p, size, 0xBC); + HwasanThread *t = GetCurrentThread(); + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocator.Deallocate(cache, p); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocator.Deallocate(cache, p); + } +} + +void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size, + uptr alignment) { + alignment = Max(alignment, kShadowAlignment); + new_size = RoundUpTo(new_size, kShadowAlignment); + + void *old_p = GetAddressFromPointer(user_old_p); + Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p)); + uptr old_size = meta->requested_size; + uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); + if (new_size <= actually_allocated_size) { + // We are not reallocating here. + // FIXME: update stack trace for the allocation? + meta->requested_size = new_size; + if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) + return user_old_p; + if (flags()->retag_in_realloc) + return (void *)TagMemoryAligned((uptr)old_p, new_size, 0xCC); + if (new_size > old_size) { + tag_t tag = GetTagFromPointer((uptr)user_old_p); + TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag); + } + return user_old_p; + } + uptr memcpy_size = Min(new_size, old_size); + void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); + if (new_p) { + internal_memcpy(new_p, old_p, memcpy_size); + HwasanDeallocate(stack, old_p); + } + return new_p; +} + +HwasanChunkView FindHeapChunkByAddress(uptr address) { + void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address)); + if (!block) + return HwasanChunkView(); + Metadata *metadata = + reinterpret_cast<Metadata*>(allocator.GetMetaData(block)); + return HwasanChunkView(reinterpret_cast<uptr>(block), metadata); +} + +static uptr AllocationSize(const void *user_ptr) { + const void *p = GetAddressFromPointer(user_ptr); + if (!p) return 0; + const void *beg = allocator.GetBlockBegin(p); + if (beg != p) return 0; + Metadata *b = (Metadata *)allocator.GetMetaData(p); + return b->requested_size; +} + +void *hwasan_malloc(uptr size, StackTrace *stack) { + return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); +} + +void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) + return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest()); + return SetErrnoOnNull(HwasanAllocate(stack, nmemb * size, sizeof(u64), true)); +} + +void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { + if (!ptr) + return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + if (size == 0) { + HwasanDeallocate(stack, ptr); + return nullptr; + } + return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); +} + +void *hwasan_valloc(uptr size, StackTrace *stack) { + return SetErrnoOnNull(HwasanAllocate(stack, size, GetPageSizeCached(), false)); +} + +void *hwasan_pvalloc(uptr size, StackTrace *stack) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + return Allocator::FailureHandler::OnBadRequest(); + } + // pvalloc(0) should allocate one page. + size = size ? RoundUpTo(size, PageSize) : PageSize; + return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false)); +} + +void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); +} + +void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) { + if (UNLIKELY(!IsPowerOfTwo(alignment))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); +} + +int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, + StackTrace *stack) { + if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { + Allocator::FailureHandler::OnBadRequest(); + return errno_EINVAL; + } + void *ptr = HwasanAllocate(stack, size, alignment, false); + if (UNLIKELY(!ptr)) + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, alignment)); + *memptr = ptr; + return 0; +} + +} // namespace __hwasan + +using namespace __hwasan; + +void __hwasan_enable_allocator_tagging() { + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1); +} + +void __hwasan_disable_allocator_tagging() { + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0); +} + +uptr __sanitizer_get_current_allocated_bytes() { + uptr stats[AllocatorStatCount]; + allocator.GetStats(stats); + return stats[AllocatorStatAllocated]; +} + +uptr __sanitizer_get_heap_size() { + uptr stats[AllocatorStatCount]; + allocator.GetStats(stats); + return stats[AllocatorStatMapped]; +} + +uptr __sanitizer_get_free_bytes() { return 1; } + +uptr __sanitizer_get_unmapped_bytes() { return 1; } + +uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } + +int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; } + +uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h new file mode 100644 index 000000000000..d025112e9773 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -0,0 +1,55 @@ +//===-- hwasan_allocator.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_ALLOCATOR_H +#define HWASAN_ALLOCATOR_H + +#include "sanitizer_common/sanitizer_common.h" + +namespace __hwasan { + +struct HwasanThreadLocalMallocStorage { + uptr quarantine_cache[16]; + // Allocator cache contains atomic_uint64_t which must be 8-byte aligned. + ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque. + void CommitBack(); + + private: + // These objects are allocated via mmap() and are zero-initialized. + HwasanThreadLocalMallocStorage() {} +}; + +struct Metadata; + +class HwasanChunkView { + public: + HwasanChunkView() : block_(0), metadata_(nullptr) {} + HwasanChunkView(uptr block, Metadata *metadata) + : block_(block), metadata_(metadata) {} + bool IsValid() const; // Checks if it points to a valid allocated chunk + bool IsAllocated() const; // Checks if the memory is currently allocated + uptr Beg() const; // First byte of user memory + uptr End() const; // Last byte of user memory + uptr UsedSize() const; // Size requested by the user + u32 GetAllocStackId() const; + u32 GetFreeStackId() const; + private: + uptr block_; + Metadata *const metadata_; +}; + +HwasanChunkView FindHeapChunkByAddress(uptr address); + +} // namespace __hwasan + +#endif // HWASAN_ALLOCATOR_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt b/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt new file mode 100644 index 000000000000..395ba28f0212 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt @@ -0,0 +1,7 @@ +# Blacklist for HWAddressSanitizer. Turns off instrumentation of particular +# functions or sources. Use with care. You may set location of blacklist +# at compile-time using -fsanitize-blacklist=<path> flag. + +# Example usage: +# fun:*bad_function_name* +# src:file_with_tricky_code.cc diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.h b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h new file mode 100644 index 000000000000..16d60c4d8ba5 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h @@ -0,0 +1,30 @@ +//===-- hwasan_flags.h --------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// +#ifndef HWASAN_FLAGS_H +#define HWASAN_FLAGS_H + +namespace __hwasan { + +struct Flags { +#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "hwasan_flags.inc" +#undef HWASAN_FLAG + + void SetDefaults(); +}; + +Flags *flags(); + +} // namespace __hwasan + +#endif // HWASAN_FLAGS_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc new file mode 100644 index 000000000000..a2cb701ae93d --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc @@ -0,0 +1,29 @@ +//===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Hwasan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef HWASAN_FLAG +# error "Define HWASAN_FLAG prior to including this file!" +#endif + +// HWASAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +HWASAN_FLAG(bool, tag_in_malloc, true, "") +HWASAN_FLAG(bool, tag_in_free, true, "") +HWASAN_FLAG(bool, retag_in_realloc, true, "") +HWASAN_FLAG(bool, print_stats, false, "") +HWASAN_FLAG(bool, halt_on_error, true, "") +HWASAN_FLAG(bool, atexit, false, "") + +// Test only flag to disable malloc/realloc/free memory tagging on startup. +// Tagging can be reenabled with __hwasan_enable_allocator_tagging(). +HWASAN_FLAG(bool, disable_allocator_tagging, false, "") diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc new file mode 100644 index 000000000000..fb39e9fdacf9 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc @@ -0,0 +1,483 @@ +//===-- hwasan_interceptors.cc ----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Interceptors for standard library functions. +// +// FIXME: move as many interceptors as possible into +// sanitizer_common/sanitizer_common_interceptors.h +//===----------------------------------------------------------------------===// + +#include "interception/interception.h" +#include "hwasan.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +#include <stdarg.h> +// ACHTUNG! No other system header includes in this file. +// Ideally, we should get rid of stdarg.h as well. + +using namespace __hwasan; + +using __sanitizer::memory_order; +using __sanitizer::atomic_load; +using __sanitizer::atomic_store; +using __sanitizer::atomic_uintptr_t; + +DECLARE_REAL(SIZE_T, strlen, const char *s) +DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen) +DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n) +DECLARE_REAL(void *, memset, void *dest, int c, uptr n) + +bool IsInInterceptorScope() { + HwasanThread *t = GetCurrentThread(); + return t && t->InInterceptorScope(); +} + +struct InterceptorScope { + InterceptorScope() { + HwasanThread *t = GetCurrentThread(); + if (t) + t->EnterInterceptorScope(); + } + ~InterceptorScope() { + HwasanThread *t = GetCurrentThread(); + if (t) + t->LeaveInterceptorScope(); + } +}; + +static uptr allocated_for_dlsym; +static const uptr kDlsymAllocPoolSize = 1024; +static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; + +static bool IsInDlsymAllocPool(const void *ptr) { + uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + return off < sizeof(alloc_memory_for_dlsym); +} + +static void *AllocateFromLocalPool(uptr size_in_bytes) { + uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; + void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; + allocated_for_dlsym += size_in_words; + CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); + return mem; +} + +#define ENSURE_HWASAN_INITED() do { \ + CHECK(!hwasan_init_is_running); \ + if (!hwasan_inited) { \ + __hwasan_init(); \ + } \ +} while (0) + + + +#define HWASAN_READ_RANGE(ctx, offset, size) \ + CHECK_UNPOISONED(offset, size) +#define HWASAN_WRITE_RANGE(ctx, offset, size) \ + CHECK_UNPOISONED(offset, size) + + + +// Check that [x, x+n) range is unpoisoned. +#define CHECK_UNPOISONED_0(x, n) \ + do { \ + sptr __offset = __hwasan_test_shadow(x, n); \ + if (__hwasan::IsInSymbolizer()) break; \ + if (__offset >= 0) { \ + GET_CALLER_PC_BP_SP; \ + (void)sp; \ + ReportInvalidAccessInsideAddressRange(__func__, x, n, __offset); \ + __hwasan::PrintWarning(pc, bp); \ + if (__hwasan::flags()->halt_on_error) { \ + Printf("Exiting\n"); \ + Die(); \ + } \ + } \ + } while (0) + +// Check that [x, x+n) range is unpoisoned unless we are in a nested +// interceptor. +#define CHECK_UNPOISONED(x, n) \ + do { \ + if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \ + } while (0) + +#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \ + CHECK_UNPOISONED((x), \ + common_flags()->strict_string_checks ? (len) + 1 : (n) ) + + +INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + CHECK_NE(memptr, 0); + int res = hwasan_posix_memalign(memptr, alignment, size, &stack); + return res; +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_memalign(alignment, size, &stack); +} +#define HWASAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign) +#else +#define HWASAN_MAYBE_INTERCEPT_MEMALIGN +#endif + +INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_aligned_alloc(alignment, size, &stack); +} + +INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + void *ptr = hwasan_memalign(alignment, size, &stack); + if (ptr) + DTLS_on_libc_memalign(ptr, size); + return ptr; +} + +INTERCEPTOR(void *, valloc, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_valloc(size, &stack); +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void *, pvalloc, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_pvalloc(size, &stack); +} +#define HWASAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc) +#else +#define HWASAN_MAYBE_INTERCEPT_PVALLOC +#endif + +INTERCEPTOR(void, free, void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + HwasanDeallocate(&stack, ptr); +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void, cfree, void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + HwasanDeallocate(&stack, ptr); +} +#define HWASAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree) +#else +#define HWASAN_MAYBE_INTERCEPT_CFREE +#endif + +INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { + return __sanitizer_get_allocated_size(ptr); +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +// This function actually returns a struct by value, but we can't unpoison a +// temporary! The following is equivalent on all supported platforms but +// aarch64 (which uses a different register for sret value). We have a test +// to confirm that. +INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { +#ifdef __aarch64__ + uptr r8; + asm volatile("mov %0,x8" : "=r" (r8)); + sret = reinterpret_cast<__sanitizer_mallinfo*>(r8); +#endif + REAL(memset)(sret, 0, sizeof(*sret)); +} +#define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo) +#else +#define HWASAN_MAYBE_INTERCEPT_MALLINFO +#endif + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(int, mallopt, int cmd, int value) { + return -1; +} +#define HWASAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) +#else +#define HWASAN_MAYBE_INTERCEPT_MALLOPT +#endif + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void, malloc_stats, void) { + // FIXME: implement, but don't call REAL(malloc_stats)! +} +#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS INTERCEPT_FUNCTION(malloc_stats) +#else +#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS +#endif + + +INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. + return AllocateFromLocalPool(nmemb * size); + return hwasan_calloc(nmemb, size, &stack); +} + +INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(IsInDlsymAllocPool(ptr))) { + uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); + void *new_ptr; + if (UNLIKELY(!hwasan_inited)) { + new_ptr = AllocateFromLocalPool(copy_size); + } else { + copy_size = size; + new_ptr = hwasan_malloc(copy_size, &stack); + } + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; + } + return hwasan_realloc(ptr, size, &stack); +} + +INTERCEPTOR(void *, malloc, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. + return AllocateFromLocalPool(size); + return hwasan_malloc(size, &stack); +} + + +INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags, + int fd, OFF_T offset) { + if (hwasan_init_is_running) + return REAL(mmap)(addr, length, prot, flags, fd, offset); + ENSURE_HWASAN_INITED(); + if (addr && !MEM_IS_APP(addr)) { + if (flags & map_fixed) { + errno = errno_EINVAL; + return (void *)-1; + } else { + addr = nullptr; + } + } + void *res = REAL(mmap)(addr, length, prot, flags, fd, offset); + return res; +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags, + int fd, OFF64_T offset) { + ENSURE_HWASAN_INITED(); + if (addr && !MEM_IS_APP(addr)) { + if (flags & map_fixed) { + errno = errno_EINVAL; + return (void *)-1; + } else { + addr = nullptr; + } + } + void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset); + return res; +} +#define HWASAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64) +#else +#define HWASAN_MAYBE_INTERCEPT_MMAP64 +#endif + +extern "C" int pthread_attr_init(void *attr); +extern "C" int pthread_attr_destroy(void *attr); + +static void *HwasanThreadStartFunc(void *arg) { + HwasanThread *t = (HwasanThread *)arg; + SetCurrentThread(t); + return t->ThreadStart(); +} + +INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), + void * param) { + ENSURE_HWASAN_INITED(); // for GetTlsSize() + __sanitizer_pthread_attr_t myattr; + if (!attr) { + pthread_attr_init(&myattr); + attr = &myattr; + } + + AdjustStackSize(attr); + + HwasanThread *t = HwasanThread::Create(callback, param); + + int res = REAL(pthread_create)(th, attr, HwasanThreadStartFunc, t); + + if (attr == &myattr) + pthread_attr_destroy(&myattr); + return res; +} + +static void BeforeFork() { + StackDepotLockAll(); +} + +static void AfterFork() { + StackDepotUnlockAll(); +} + +INTERCEPTOR(int, fork, void) { + ENSURE_HWASAN_INITED(); + BeforeFork(); + int pid = REAL(fork)(); + AfterFork(); + return pid; +} + + +struct HwasanInterceptorContext { + bool in_interceptor_scope; +}; + +namespace __hwasan { + +int OnExit() { + // FIXME: ask frontend whether we need to return failure. + return 0; +} + +} // namespace __hwasan + +// A version of CHECK_UNPOISONED using a saved scope value. Used in common +// interceptors. +#define CHECK_UNPOISONED_CTX(ctx, x, n) \ + do { \ + if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \ + CHECK_UNPOISONED_0(x, n); \ + } while (0) + +#define HWASAN_INTERCEPT_FUNC(name) \ + do { \ + if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ + VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \ + } while (0) + +#define HWASAN_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ + VReport( \ + 1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ + } while (0) + +#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name) +#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ + HWASAN_INTERCEPT_FUNC_VER(name, ver) +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + CHECK_UNPOISONED_CTX(ctx, ptr, size) +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + CHECK_UNPOISONED_CTX(ctx, ptr, size) +#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \ + HWASAN_WRITE_RANGE(ctx, ptr, size) +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__); \ + ENSURE_HWASAN_INITED(); \ + HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()}; \ + ctx = (void *)&hwasan_ctx; \ + (void)ctx; \ + InterceptorScope interceptor_scope; +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ + do { \ + } while (false) // FIXME +#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ + do { \ + } while (false) // FIXME +#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) +#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() + +#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ + if (HwasanThread *t = GetCurrentThread()) { \ + *begin = t->tls_begin(); \ + *end = t->tls_end(); \ + } else { \ + *begin = *end = 0; \ + } + +#include "sanitizer_common/sanitizer_platform_interceptors.h" +#include "sanitizer_common/sanitizer_common_interceptors.inc" +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s) +#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#include "sanitizer_common/sanitizer_common_syscalls.inc" + + + +namespace __hwasan { + +void InitializeInterceptors() { + static int inited = 0; + CHECK_EQ(inited, 0); + InitializeCommonInterceptors(); + InitializeSignalInterceptors(); + + INTERCEPT_FUNCTION(mmap); + HWASAN_MAYBE_INTERCEPT_MMAP64; + INTERCEPT_FUNCTION(posix_memalign); + HWASAN_MAYBE_INTERCEPT_MEMALIGN; + INTERCEPT_FUNCTION(__libc_memalign); + INTERCEPT_FUNCTION(valloc); + HWASAN_MAYBE_INTERCEPT_PVALLOC; + INTERCEPT_FUNCTION(malloc); + INTERCEPT_FUNCTION(calloc); + INTERCEPT_FUNCTION(realloc); + INTERCEPT_FUNCTION(free); + HWASAN_MAYBE_INTERCEPT_CFREE; + INTERCEPT_FUNCTION(malloc_usable_size); + HWASAN_MAYBE_INTERCEPT_MALLINFO; + HWASAN_MAYBE_INTERCEPT_MALLOPT; + HWASAN_MAYBE_INTERCEPT_MALLOC_STATS; + INTERCEPT_FUNCTION(pthread_create); + INTERCEPT_FUNCTION(fork); + + inited = 1; +} +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h new file mode 100644 index 000000000000..08b77534e0ae --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -0,0 +1,97 @@ +//===-- hwasan_interface_internal.h -------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Private Hwasan interface header. +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_INTERFACE_INTERNAL_H +#define HWASAN_INTERFACE_INTERNAL_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_init(); + +using __sanitizer::uptr; +using __sanitizer::sptr; +using __sanitizer::uu64; +using __sanitizer::uu32; +using __sanitizer::uu16; +using __sanitizer::u64; +using __sanitizer::u32; +using __sanitizer::u16; +using __sanitizer::u8; + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load(uptr, uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load1(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load2(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load4(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load8(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load16(uptr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store(uptr, uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store1(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store2(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store4(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store8(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store16(uptr); + +// Returns the offset of the first tag mismatch or -1 if the whole range is +// good. +SANITIZER_INTERFACE_ATTRIBUTE +sptr __hwasan_test_shadow(const void *x, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +/* OPTIONAL */ const char* __hwasan_default_options(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_print_shadow(const void *x, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +u16 __sanitizer_unaligned_load16(const uu16 *p); + +SANITIZER_INTERFACE_ATTRIBUTE +u32 __sanitizer_unaligned_load32(const uu32 *p); + +SANITIZER_INTERFACE_ATTRIBUTE +u64 __sanitizer_unaligned_load64(const uu64 *p); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store16(uu16 *p, u16 x); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store32(uu32 *p, u32 x); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store64(uu64 *p, u64 x); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_enable_allocator_tagging(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_disable_allocator_tagging(); + +} // extern "C" + +#endif // HWASAN_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc new file mode 100644 index 000000000000..9b8613171cbd --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc @@ -0,0 +1,251 @@ +//===-- hwasan_linux.cc -----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Linux-, NetBSD- and FreeBSD-specific code. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD + +#include "hwasan.h" +#include "hwasan_thread.h" + +#include <elf.h> +#include <link.h> +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> +#include <unistd.h> +#include <unwind.h> +#include <sys/time.h> +#include <sys/resource.h> + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_procmaps.h" + +namespace __hwasan { + +void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { + CHECK_EQ((beg % GetMmapGranularity()), 0); + CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); + uptr size = end - beg + 1; + DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. + void *res = MmapFixedNoReserve(beg, size, name); + if (res != (void *)beg) { + Report( + "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " + "Perhaps you're using ulimit -v\n", + size); + Abort(); + } + if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size); +} + +static void ProtectGap(uptr addr, uptr size) { + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + // A few pages at the start of the address space can not be protected. + // But we really want to protect as much as possible, to prevent this memory + // being returned as a result of a non-FIXED mmap(). + if (addr == 0) { + uptr step = GetMmapGranularity(); + while (size > step) { + addr += step; + size -= step; + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + } + } + + Report( + "ERROR: Failed to protect the shadow gap. " + "ASan cannot proceed correctly. ABORTING.\n"); + DumpProcessMap(); + Die(); +} + +bool InitShadow() { + const uptr maxVirtualAddress = GetMaxUserVirtualAddress(); + + // LowMem covers as much of the first 4GB as possible. + const uptr kLowMemEnd = 1UL<<32; + const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale; + const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale; + + // HighMem covers the upper part of the address space. + const uptr kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1; + const uptr kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale); + CHECK(kHighShadowStart < kHighShadowEnd); + + const uptr kHighMemStart = kHighShadowStart << kShadowScale; + CHECK(kHighShadowEnd <= kHighMemStart); + + if (Verbosity()) { + Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemStart, + (void *)maxVirtualAddress); + if (kHighMemStart > kHighShadowEnd) + Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd, + (void *)kHighMemStart); + Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowStart, + (void *)kHighShadowEnd); + if (kHighShadowStart > kLowMemEnd) + Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd, + (void *)kHighMemStart); + Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowShadowEnd, + (void *)kLowMemEnd); + Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowStart, + (void *)kLowShadowEnd); + Printf("|| `[%p, %p]` || ShadowGap1 ||\n", (void *)0, + (void *)kLowShadowStart); + } + + ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd - 1, "low shadow"); + ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd - 1, "high shadow"); + ProtectGap(0, kLowShadowStart); + if (kHighShadowStart > kLowMemEnd) + ProtectGap(kLowMemEnd, kHighShadowStart - kLowMemEnd); + if (kHighMemStart > kHighShadowEnd) + ProtectGap(kHighShadowEnd, kHighMemStart - kHighShadowEnd); + + return true; +} + +static void HwasanAtExit(void) { + if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) + ReportStats(); + if (hwasan_report_count > 0) { + // ReportAtExitStatistics(); + if (common_flags()->exitcode) + internal__exit(common_flags()->exitcode); + } +} + +void InstallAtExitHandler() { + atexit(HwasanAtExit); +} + +// ---------------------- TSD ---------------- {{{1 + +static pthread_key_t tsd_key; +static bool tsd_key_inited = false; + +void HwasanTSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_key_inited); + tsd_key_inited = true; + CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); +} + +HwasanThread *GetCurrentThread() { + return (HwasanThread*)pthread_getspecific(tsd_key); +} + +void SetCurrentThread(HwasanThread *t) { + // Make sure that HwasanTSDDtor gets called at the end. + CHECK(tsd_key_inited); + // Make sure we do not reset the current HwasanThread. + CHECK_EQ(0, pthread_getspecific(tsd_key)); + pthread_setspecific(tsd_key, (void *)t); +} + +void HwasanTSDDtor(void *tsd) { + HwasanThread *t = (HwasanThread*)tsd; + if (t->destructor_iterations_ > 1) { + t->destructor_iterations_--; + CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); + return; + } + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + HwasanThread::TSDDtor(tsd); +} + +struct AccessInfo { + uptr addr; + uptr size; + bool is_store; + bool is_load; +}; + +#if defined(__aarch64__) +static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { + // Access type is encoded in HLT immediate as 0x1XY, + // where X is 1 for store, 0 for load. + // Valid values of Y are 0 to 4, which are interpreted as log2(access_size), + // and 0xF, which means that access size is stored in X1 register. + // Access address is always in X0 register. + AccessInfo ai; + uptr pc = (uptr)info->si_addr; + unsigned code = ((*(u32 *)pc) >> 5) & 0xffff; + if ((code & 0xff00) != 0x100) + return AccessInfo{0, 0, false, false}; // Not ours. + bool is_store = code & 0x10; + unsigned size_log = code & 0xff; + if (size_log > 4 && size_log != 0xf) + return AccessInfo{0, 0, false, false}; // Not ours. + + ai.is_store = is_store; + ai.is_load = !is_store; + ai.addr = uc->uc_mcontext.regs[0]; + if (size_log == 0xf) + ai.size = uc->uc_mcontext.regs[1]; + else + ai.size = 1U << size_log; + return ai; +} +#else +static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { + return AccessInfo{0, 0, false, false}; +} +#endif + +static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) { + SignalContext sig{info, uc}; + AccessInfo ai = GetAccessInfo(info, uc); + if (!ai.is_store && !ai.is_load) + return false; + + InternalScopedBuffer<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc, + common_flags()->fast_unwind_on_fatal); + + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store); + + ++hwasan_report_count; + if (flags()->halt_on_error) + Die(); + + uc->uc_mcontext.pc += 4; + return true; +} + +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +void HwasanOnDeadlySignal(int signo, void *info, void *context) { + // Probably a tag mismatch. + if (signo == SIGILL) + if (HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context)) + return; + + HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); +} + + +} // namespace __hwasan + +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc new file mode 100644 index 000000000000..3ccc26734bf6 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc @@ -0,0 +1,66 @@ +//===-- hwasan_new_delete.cc ------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Interceptors for operators new and delete. +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator.h" + +#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE + +#include <stddef.h> + +using namespace __hwasan; // NOLINT + +// Fake std::nothrow_t to avoid including <new>. +namespace std { + struct nothrow_t {}; +} // namespace std + + +// TODO(alekseys): throw std::bad_alloc instead of dying on OOM. +#define OPERATOR_NEW_BODY(nothrow) \ + GET_MALLOC_STACK_TRACE; \ + void *res = hwasan_malloc(size, &stack);\ + if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\ + return res + +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(true /*nothrow*/); +} +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(true /*nothrow*/); +} + +#define OPERATOR_DELETE_BODY \ + GET_MALLOC_STACK_TRACE; \ + if (ptr) HwasanDeallocate(&stack, ptr) + +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::nothrow_t const&) { + OPERATOR_DELETE_BODY; +} + +#endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc new file mode 100644 index 000000000000..411fd05b10d5 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc @@ -0,0 +1,36 @@ +//===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "hwasan_poisoning.h" + +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_common.h" + +namespace __hwasan { + +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MEM_TO_SHADOW(p); + uptr shadow_size = MEM_TO_SHADOW_OFFSET(size); + internal_memset((void *)shadow_start, tag, shadow_size); + return AddTagToPointer(p, tag); +} + +uptr TagMemory(uptr p, uptr size, tag_t tag) { + uptr start = RoundDownTo(p, kShadowAlignment); + uptr end = RoundUpTo(p + size, kShadowAlignment); + return TagMemoryAligned(start, end - start, tag); +} + +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h new file mode 100644 index 000000000000..b44a91f975f5 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h @@ -0,0 +1,25 @@ +//===-- hwasan_poisoning.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_POISONING_H +#define HWASAN_POISONING_H + +#include "hwasan.h" + +namespace __hwasan { +uptr TagMemory(uptr p, uptr size, tag_t tag); +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag); + +} // namespace __hwasan + +#endif // HWASAN_POISONING_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_report.cc b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc new file mode 100644 index 000000000000..a3c6709491d3 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc @@ -0,0 +1,133 @@ +//===-- hwasan_report.cc ----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Error reporting. +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "hwasan_allocator.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_mutex.h" +#include "sanitizer_common/sanitizer_report_decorator.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_symbolizer.h" + +using namespace __sanitizer; + +namespace __hwasan { + +static StackTrace GetStackTraceFromId(u32 id) { + CHECK(id); + StackTrace res = StackDepotGet(id); + CHECK(res.trace); + return res; +} + +class Decorator: public __sanitizer::SanitizerCommonDecorator { + public: + Decorator() : SanitizerCommonDecorator() { } + const char *Allocation() { return Magenta(); } + const char *Origin() { return Magenta(); } + const char *Name() { return Green(); } +}; + +struct HeapAddressDescription { + uptr addr; + u32 alloc_stack_id; + u32 free_stack_id; + + void Print() const { + Decorator d; + if (free_stack_id) { + Printf("%sfreed here:%s\n", d.Allocation(), d.Default()); + GetStackTraceFromId(free_stack_id).Print(); + Printf("%spreviously allocated here:%s\n", d.Allocation(), d.Default()); + } else { + Printf("%sallocated here:%s\n", d.Allocation(), d.Default()); + } + GetStackTraceFromId(alloc_stack_id).Print(); + } +}; + +bool GetHeapAddressInformation(uptr addr, uptr access_size, + HeapAddressDescription *description) { + HwasanChunkView chunk = FindHeapChunkByAddress(addr); + if (!chunk.IsValid()) + return false; + description->addr = addr; + description->alloc_stack_id = chunk.GetAllocStackId(); + description->free_stack_id = chunk.GetFreeStackId(); + return true; +} + +void PrintAddressDescription(uptr addr, uptr access_size) { + HeapAddressDescription heap_description; + if (GetHeapAddressInformation(addr, access_size, &heap_description)) { + heap_description.Print(); + return; + } + // We exhausted our possibilities. Bail out. + Printf("HWAddressSanitizer can not describe address in more detail.\n"); +} + +void ReportInvalidAccess(StackTrace *stack, u32 origin) { + ScopedErrorReportLock l; + + Decorator d; + Printf("%s", d.Warning()); + Report("WARNING: HWAddressSanitizer: invalid access\n"); + Printf("%s", d.Default()); + stack->Print(); + ReportErrorSummary("invalid-access", stack); +} + +void ReportStats() {} + +void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, + uptr size, uptr offset) { + ScopedErrorReportLock l; + + Decorator d; + Printf("%s", d.Warning()); + Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n", + d.Warning(), d.Name(), what, d.Warning(), offset, start, size, + d.Default()); + PrintAddressDescription((uptr)start + offset, 1); + // if (__sanitizer::Verbosity()) + // DescribeMemoryRange(start, size); +} + +void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, + bool is_store) { + ScopedErrorReportLock l; + + Decorator d; + Printf("%s", d.Warning()); + uptr address = GetAddressFromPointer(addr); + Printf("%s of size %zu at %p\n", is_store ? "WRITE" : "READ", access_size, + address); + + tag_t ptr_tag = GetTagFromPointer(addr); + tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(address); + Printf("pointer tag 0x%x\nmemory tag 0x%x\n", ptr_tag, mem_tag); + Printf("%s", d.Default()); + + stack->Print(); + + PrintAddressDescription(address, access_size); + + ReportErrorSummary("tag-mismatch", stack); +} + + +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc new file mode 100644 index 000000000000..d6551ffc6fdc --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc @@ -0,0 +1,75 @@ + +#include "hwasan.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" +#include "hwasan_interface_internal.h" + +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +namespace __hwasan { + +HwasanThread *HwasanThread::Create(thread_callback_t start_routine, + void *arg) { + uptr PageSize = GetPageSizeCached(); + uptr size = RoundUpTo(sizeof(HwasanThread), PageSize); + HwasanThread *thread = (HwasanThread*)MmapOrDie(size, __func__); + thread->start_routine_ = start_routine; + thread->arg_ = arg; + thread->destructor_iterations_ = GetPthreadDestructorIterations(); + + return thread; +} + +void HwasanThread::SetThreadStackAndTls() { + uptr tls_size = 0; + uptr stack_size = 0; + GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, + &tls_begin_, &tls_size); + stack_top_ = stack_bottom_ + stack_size; + tls_end_ = tls_begin_ + tls_size; + + int local; + CHECK(AddrIsInStack((uptr)&local)); +} + +void HwasanThread::Init() { + SetThreadStackAndTls(); + CHECK(MEM_IS_APP(stack_bottom_)); + CHECK(MEM_IS_APP(stack_top_ - 1)); +} + +void HwasanThread::TSDDtor(void *tsd) { + HwasanThread *t = (HwasanThread*)tsd; + t->Destroy(); +} + +void HwasanThread::ClearShadowForThreadStackAndTLS() { + TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); + if (tls_begin_ != tls_end_) + TagMemory(tls_begin_, tls_end_ - tls_begin_, 0); +} + +void HwasanThread::Destroy() { + malloc_storage().CommitBack(); + ClearShadowForThreadStackAndTLS(); + uptr size = RoundUpTo(sizeof(HwasanThread), GetPageSizeCached()); + UnmapOrDie(this, size); + DTLS_Destroy(); +} + +thread_return_t HwasanThread::ThreadStart() { + Init(); + + if (!start_routine_) { + // start_routine_ == 0 if we're on the main thread or on one of the + // OS X libdispatch worker threads. But nobody is supposed to call + // ThreadStart() for the worker threads. + return 0; + } + + thread_return_t res = start_routine_(arg_); + + return res; +} + +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.h b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h new file mode 100644 index 000000000000..96f1bb813adf --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h @@ -0,0 +1,81 @@ +//===-- hwasan_thread.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_THREAD_H +#define HWASAN_THREAD_H + +#include "hwasan_allocator.h" +#include "sanitizer_common/sanitizer_common.h" + +namespace __hwasan { + +class HwasanThread { + public: + static HwasanThread *Create(thread_callback_t start_routine, void *arg); + static void TSDDtor(void *tsd); + void Destroy(); + + void Init(); // Should be called from the thread itself. + thread_return_t ThreadStart(); + + uptr stack_top() { return stack_top_; } + uptr stack_bottom() { return stack_bottom_; } + uptr tls_begin() { return tls_begin_; } + uptr tls_end() { return tls_end_; } + bool IsMainThread() { return start_routine_ == nullptr; } + + bool AddrIsInStack(uptr addr) { + return addr >= stack_bottom_ && addr < stack_top_; + } + + bool InSignalHandler() { return in_signal_handler_; } + void EnterSignalHandler() { in_signal_handler_++; } + void LeaveSignalHandler() { in_signal_handler_--; } + + bool InSymbolizer() { return in_symbolizer_; } + void EnterSymbolizer() { in_symbolizer_++; } + void LeaveSymbolizer() { in_symbolizer_--; } + + bool InInterceptorScope() { return in_interceptor_scope_; } + void EnterInterceptorScope() { in_interceptor_scope_++; } + void LeaveInterceptorScope() { in_interceptor_scope_--; } + + HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } + + int destructor_iterations_; + + private: + // NOTE: There is no HwasanThread constructor. It is allocated + // via mmap() and *must* be valid in zero-initialized state. + void SetThreadStackAndTls(); + void ClearShadowForThreadStackAndTLS(); + thread_callback_t start_routine_; + void *arg_; + uptr stack_top_; + uptr stack_bottom_; + uptr tls_begin_; + uptr tls_end_; + + unsigned in_signal_handler_; + unsigned in_symbolizer_; + unsigned in_interceptor_scope_; + + HwasanThreadLocalMallocStorage malloc_storage_; +}; + +HwasanThread *GetCurrentThread(); +void SetCurrentThread(HwasanThread *t); + +} // namespace __hwasan + +#endif // HWASAN_THREAD_H diff --git a/contrib/compiler-rt/lib/interception/interception.h b/contrib/compiler-rt/lib/interception/interception.h index d79fa67babfa..cba484936eac 100644 --- a/contrib/compiler-rt/lib/interception/interception.h +++ b/contrib/compiler-rt/lib/interception/interception.h @@ -15,13 +15,14 @@ #ifndef INTERCEPTION_H #define INTERCEPTION_H -#if !defined(__linux__) && !defined(__FreeBSD__) && \ - !defined(__APPLE__) && !defined(_WIN32) +#include "sanitizer_common/sanitizer_internal_defs.h" + +#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \ + !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \ + !SANITIZER_SOLARIS # error "Interception doesn't work on this operating system." #endif -#include "sanitizer_common/sanitizer_internal_defs.h" - // These typedefs should be used only in the interceptor definitions to replace // the standard system types (e.g. SSIZE_T instead of ssize_t) typedef __sanitizer::uptr SIZE_T; @@ -87,7 +88,7 @@ typedef __sanitizer::OFF64_T OFF64_T; // As it's decided at compile time which functions are to be intercepted on Mac, // INTERCEPT_FUNCTION() is effectively a no-op on this system. -#if defined(__APPLE__) +#if SANITIZER_MAC #include <sys/cdefs.h> // For __DARWIN_ALIAS_C(). // Just a pair of pointers. @@ -121,7 +122,7 @@ const interpose_substitution substitution_##func_name[] \ # define INTERCEPTOR_ATTRIBUTE # define DECLARE_WRAPPER(ret_type, func, ...) -#elif defined(_WIN32) +#elif SANITIZER_WINDOWS # define WRAP(x) __asan_wrap_##x # define WRAPPER_NAME(x) "__asan_wrap_"#x # define INTERCEPTOR_ATTRIBUTE __declspec(dllexport) @@ -129,7 +130,7 @@ const interpose_substitution substitution_##func_name[] \ extern "C" ret_type func(__VA_ARGS__); # define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__); -#elif defined(__FreeBSD__) +#elif SANITIZER_FREEBSD || SANITIZER_NETBSD # define WRAP(x) __interceptor_ ## x # define WRAPPER_NAME(x) "__interceptor_" #x # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) @@ -139,7 +140,7 @@ const interpose_substitution substitution_##func_name[] \ # define DECLARE_WRAPPER(ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__) \ __attribute__((alias("__interceptor_" #func), visibility("default"))); -#else +#elif !SANITIZER_FUCHSIA # define WRAP(x) __interceptor_ ## x # define WRAPPER_NAME(x) "__interceptor_" #x # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) @@ -148,7 +149,15 @@ const interpose_substitution substitution_##func_name[] \ __attribute__((weak, alias("__interceptor_" #func), visibility("default"))); #endif -#if !defined(__APPLE__) +#if SANITIZER_FUCHSIA +// There is no general interception at all on Fuchsia. +// Sanitizer runtimes just define functions directly to preempt them, +// and have bespoke ways to access the underlying libc functions. +# include <zircon/sanitizer.h> +# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) +# define REAL(x) __unsanitized_##x +# define DECLARE_REAL(ret_type, func, ...) +#elif !SANITIZER_MAC # define PTR_TO_REAL(x) real_##x # define REAL(x) __interception::PTR_TO_REAL(x) # define FUNC_TYPE(x) x##_f @@ -159,22 +168,26 @@ const interpose_substitution substitution_##func_name[] \ extern FUNC_TYPE(func) PTR_TO_REAL(func); \ } # define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src) -#else // __APPLE__ +#else // SANITIZER_MAC # define REAL(x) x # define DECLARE_REAL(ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__); # define ASSIGN_REAL(x, y) -#endif // __APPLE__ +#endif // SANITIZER_MAC +#if !SANITIZER_FUCHSIA #define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ DECLARE_REAL(ret_type, func, __VA_ARGS__) \ extern "C" ret_type WRAP(func)(__VA_ARGS__); +#else +#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) +#endif // Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR // macros does its job. In exceptional cases you may need to call REAL(foo) // without defining INTERCEPTOR(..., foo, ...). For example, if you override // foo with an interceptor for other function. -#if !defined(__APPLE__) +#if !SANITIZER_MAC && !SANITIZER_FUCHSIA # define DEFINE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ namespace __interception { \ @@ -184,7 +197,18 @@ const interpose_substitution substitution_##func_name[] \ # define DEFINE_REAL(ret_type, func, ...) #endif -#if !defined(__APPLE__) +#if SANITIZER_FUCHSIA + +// We need to define the __interceptor_func name just to get +// sanitizer_common/scripts/gen_dynamic_list.py to export func. +// But we don't need to export __interceptor_func to get that. +#define INTERCEPTOR(ret_type, func, ...) \ + extern "C"[[ gnu::alias(#func), gnu::visibility("hidden") ]] ret_type \ + __interceptor_##func(__VA_ARGS__); \ + extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__) + +#elif !SANITIZER_MAC + #define INTERCEPTOR(ret_type, func, ...) \ DEFINE_REAL(ret_type, func, __VA_ARGS__) \ DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \ @@ -196,7 +220,7 @@ const interpose_substitution substitution_##func_name[] \ #define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \ INTERCEPTOR(ret_type, func, __VA_ARGS__) -#else // __APPLE__ +#else // SANITIZER_MAC #define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__) suffix; \ @@ -215,7 +239,7 @@ const interpose_substitution substitution_##func_name[] \ INTERPOSER_2(overridee, WRAP(overrider)) #endif -#if defined(_WIN32) +#if SANITIZER_WINDOWS # define INTERCEPTOR_WINAPI(ret_type, func, ...) \ typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \ namespace __interception { \ @@ -241,17 +265,19 @@ typedef unsigned long uptr; // NOLINT #define INCLUDED_FROM_INTERCEPTION_LIB -#if defined(__linux__) || defined(__FreeBSD__) +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS + # include "interception_linux.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) -#elif defined(__APPLE__) +#elif SANITIZER_MAC # include "interception_mac.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ INTERCEPT_FUNCTION_VER_MAC(func, symver) -#else // defined(_WIN32) +#elif SANITIZER_WINDOWS # include "interception_win.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_WIN(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ diff --git a/contrib/compiler-rt/lib/interception/interception_linux.cc b/contrib/compiler-rt/lib/interception/interception_linux.cc index 6e908ac017b2..c991550a4f72 100644 --- a/contrib/compiler-rt/lib/interception/interception_linux.cc +++ b/contrib/compiler-rt/lib/interception/interception_linux.cc @@ -12,25 +12,44 @@ // Linux-specific interception methods. //===----------------------------------------------------------------------===// -#if defined(__linux__) || defined(__FreeBSD__) #include "interception.h" +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS + #include <dlfcn.h> // for dlsym() and dlvsym() +#if SANITIZER_NETBSD +#include "sanitizer_common/sanitizer_libc.h" +#endif + namespace __interception { bool GetRealFunctionAddress(const char *func_name, uptr *func_addr, uptr real, uptr wrapper) { +#if SANITIZER_NETBSD + // XXX: Find a better way to handle renames + if (internal_strcmp(func_name, "sigaction") == 0) func_name = "__sigaction14"; +#endif *func_addr = (uptr)dlsym(RTLD_NEXT, func_name); + if (!*func_addr) { + // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is + // later in the library search order than the DSO that we are trying to + // intercept, which means that we cannot intercept this function. We still + // want the address of the real definition, though, so look it up using + // RTLD_DEFAULT. + *func_addr = (uptr)dlsym(RTLD_DEFAULT, func_name); + } return real == wrapper; } -#if !defined(__ANDROID__) // android does not have dlvsym +// Android and Solaris do not have dlvsym +#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS void *GetFuncAddrVer(const char *func_name, const char *ver) { return dlvsym(RTLD_NEXT, func_name, ver); } -#endif // !defined(__ANDROID__) +#endif // !SANITIZER_ANDROID } // namespace __interception - -#endif // __linux__ || __FreeBSD__ +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/interception/interception_linux.h b/contrib/compiler-rt/lib/interception/interception_linux.h index 27a66c882041..98fe51b858fc 100644 --- a/contrib/compiler-rt/lib/interception/interception_linux.h +++ b/contrib/compiler-rt/lib/interception/interception_linux.h @@ -12,7 +12,8 @@ // Linux-specific interception methods. //===----------------------------------------------------------------------===// -#if defined(__linux__) || defined(__FreeBSD__) +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_linux.h should be included from interception library only" @@ -34,14 +35,16 @@ void *GetFuncAddrVer(const char *func_name, const char *ver); (::__interception::uptr) & (func), \ (::__interception::uptr) & WRAP(func)) -#if !defined(__ANDROID__) // android does not have dlvsym +// Android and Solaris do not have dlvsym +#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ (::__interception::real_##func = (func##_f)( \ unsigned long)::__interception::GetFuncAddrVer(#func, symver)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) -#endif // !defined(__ANDROID__) +#endif // !SANITIZER_ANDROID && !SANITIZER_SOLARIS #endif // INTERCEPTION_LINUX_H -#endif // __linux__ || __FreeBSD__ +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/interception/interception_mac.cc b/contrib/compiler-rt/lib/interception/interception_mac.cc index b035cf998140..ea8072f8dd73 100644 --- a/contrib/compiler-rt/lib/interception/interception_mac.cc +++ b/contrib/compiler-rt/lib/interception/interception_mac.cc @@ -12,9 +12,8 @@ // Mac-specific interception methods. //===----------------------------------------------------------------------===// -#ifdef __APPLE__ - #include "interception.h" +#if SANITIZER_MAC -#endif // __APPLE__ +#endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/interception/interception_mac.h b/contrib/compiler-rt/lib/interception/interception_mac.h index e5a35c6971cd..6f31fed47b84 100644 --- a/contrib/compiler-rt/lib/interception/interception_mac.h +++ b/contrib/compiler-rt/lib/interception/interception_mac.h @@ -12,7 +12,7 @@ // Mac-specific interception methods. //===----------------------------------------------------------------------===// -#ifdef __APPLE__ +#if SANITIZER_MAC #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_mac.h should be included from interception.h only" @@ -25,4 +25,4 @@ #define INTERCEPT_FUNCTION_VER_MAC(func, symver) #endif // INTERCEPTION_MAC_H -#endif // __APPLE__ +#endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/interception/interception_type_test.cc b/contrib/compiler-rt/lib/interception/interception_type_test.cc index 98ce2e6afc1b..2b3a6d509bfb 100644 --- a/contrib/compiler-rt/lib/interception/interception_type_test.cc +++ b/contrib/compiler-rt/lib/interception/interception_type_test.cc @@ -12,9 +12,10 @@ // Compile-time tests of the internal type definitions. //===----------------------------------------------------------------------===// -#if defined(__linux__) || defined(__APPLE__) - #include "interception.h" + +#if SANITIZER_LINUX || SANITIZER_MAC + #include <sys/types.h> #include <stddef.h> #include <stdint.h> @@ -24,14 +25,14 @@ COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t)); COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t)); COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t)); -#ifndef __APPLE__ +#if !SANITIZER_MAC COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t)); #endif // The following are the cases when pread (and friends) is used instead of // pread64. In those cases we need OFF_T to match off_t. We don't care about the // rest (they depend on _FILE_OFFSET_BITS setting when building an application). -# if defined(__ANDROID__) || !defined _FILE_OFFSET_BITS || \ +# if SANITIZER_ANDROID || !defined _FILE_OFFSET_BITS || \ _FILE_OFFSET_BITS != 64 COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t)); # endif diff --git a/contrib/compiler-rt/lib/interception/interception_win.cc b/contrib/compiler-rt/lib/interception/interception_win.cc index b2902d57f542..dc3fe35242ef 100644 --- a/contrib/compiler-rt/lib/interception/interception_win.cc +++ b/contrib/compiler-rt/lib/interception/interception_win.cc @@ -125,9 +125,9 @@ // addr2: .bytes <body> //===----------------------------------------------------------------------===// -#ifdef _WIN32 - #include "interception.h" + +#if SANITIZER_WINDOWS #include "sanitizer_common/sanitizer_platform.h" #define WIN32_LEAN_AND_MEAN #include <windows.h> @@ -223,9 +223,8 @@ static bool IsMemoryPadding(uptr address, uptr size) { return true; } -static const u8 kHintNop10Bytes[] = { - 0x66, 0x66, 0x0F, 0x1F, 0x84, - 0x00, 0x00, 0x00, 0x00, 0x00 +static const u8 kHintNop9Bytes[] = { + 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; template<class T> @@ -240,8 +239,8 @@ static bool FunctionHasPrefix(uptr address, const T &pattern) { static bool FunctionHasPadding(uptr address, uptr size) { if (IsMemoryPadding(address - size, size)) return true; - if (size <= sizeof(kHintNop10Bytes) && - FunctionHasPrefix(address, kHintNop10Bytes)) + if (size <= sizeof(kHintNop9Bytes) && + FunctionHasPrefix(address, kHintNop9Bytes)) return true; return false; } @@ -554,7 +553,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi + case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx return 5; + case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY + return 6; } #else @@ -833,6 +835,7 @@ bool OverrideFunction( static void **InterestingDLLsAvailable() { static const char *InterestingDLLs[] = { "kernel32.dll", + "msvcr100.dll", // VS2010 "msvcr110.dll", // VS2012 "msvcr120.dll", // VS2013 "vcruntime140.dll", // VS2015 @@ -1010,4 +1013,4 @@ bool OverrideImportedFunction(const char *module_to_patch, } // namespace __interception -#endif // _WIN32 +#endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/interception/interception_win.h b/contrib/compiler-rt/lib/interception/interception_win.h index 9061f9ed4c21..71a44428f5c7 100644 --- a/contrib/compiler-rt/lib/interception/interception_win.h +++ b/contrib/compiler-rt/lib/interception/interception_win.h @@ -12,7 +12,7 @@ // Windows-specific interception methods. //===----------------------------------------------------------------------===// -#ifdef _WIN32 +#if SANITIZER_WINDOWS #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_win.h should be included from interception library only" @@ -81,4 +81,4 @@ void TestOnlyReleaseTrampolineRegions(); (::__interception::uptr *)&REAL(func)) #endif // INTERCEPTION_WIN_H -#endif // _WIN32 +#endif // SANITIZER_WINDOWS diff --git a/contrib/compiler-rt/lib/lsan/lsan.cc b/contrib/compiler-rt/lib/lsan/lsan.cc index 6c4767d61252..a9f7e399e14c 100644 --- a/contrib/compiler-rt/lib/lsan/lsan.cc +++ b/contrib/compiler-rt/lib/lsan/lsan.cc @@ -56,6 +56,9 @@ static void InitializeFlags() { RegisterLsanFlags(&parser, f); RegisterCommonFlags(&parser); + // Override from user-specified string. + const char *lsan_default_options = MaybeCallLsanDefaultOptions(); + parser.ParseString(lsan_default_options); parser.ParseString(GetEnv("LSAN_OPTIONS")); SetVerbosity(common_flags()->verbosity); @@ -65,6 +68,17 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); } +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +static void LsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind, + nullptr); +} + extern "C" void __lsan_init() { CHECK(!lsan_init_is_running); if (lsan_inited) @@ -80,6 +94,7 @@ extern "C" void __lsan_init() { InitTlsSize(); InitializeInterceptors(); InitializeThreadRegistry(); + InstallDeadlySignalHandlers(LsanOnDeadlySignal); u32 tid = ThreadCreate(0, 0, true); CHECK_EQ(tid, 0); ThreadStart(tid, GetTid()); diff --git a/contrib/compiler-rt/lib/lsan/lsan.h b/contrib/compiler-rt/lib/lsan/lsan.h index 7446e9507e0d..6baee81d00cd 100644 --- a/contrib/compiler-rt/lib/lsan/lsan.h +++ b/contrib/compiler-rt/lib/lsan/lsan.h @@ -12,24 +12,14 @@ // //===----------------------------------------------------------------------===// +#include "lsan_thread.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#define GET_STACK_TRACE(max_size, fast) \ - BufferedStackTrace stack; \ - { \ - uptr stack_top = 0, stack_bottom = 0; \ - ThreadContext *t; \ - if (fast && (t = CurrentThreadContext())) { \ - stack_top = t->stack_end(); \ - stack_bottom = t->stack_begin(); \ - } \ - if (!SANITIZER_MIPS || \ - IsValidFrame(GET_CURRENT_FRAME(), stack_top, stack_bottom)) { \ - stack.Unwind(max_size, StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ - /* context */ 0, stack_top, stack_bottom, fast); \ - } \ - } +#define GET_STACK_TRACE(max_size, fast) \ + __sanitizer::BufferedStackTrace stack; \ + GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), nullptr, fast); #define GET_STACK_TRACE_FATAL \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) @@ -51,9 +41,27 @@ void ReplaceSystemMalloc(); __lsan_init(); \ } while (0) +// Get the stack trace with the given pc and bp. +// The pc will be in the position 0 of the resulting stack trace. +// The bp may refer to the current frame or to the caller's frame. +ALWAYS_INLINE +void GetStackTrace(__sanitizer::BufferedStackTrace *stack, + __sanitizer::uptr max_depth, __sanitizer::uptr pc, + __sanitizer::uptr bp, void *context, bool fast) { + uptr stack_top = 0, stack_bottom = 0; + ThreadContext *t; + if (fast && (t = CurrentThreadContext())) { + stack_top = t->stack_end(); + stack_bottom = t->stack_begin(); + } + if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { + stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast); + } +} + } // namespace __lsan extern bool lsan_inited; extern bool lsan_init_is_running; -extern "C" void __lsan_init(); +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __lsan_init(); diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.h b/contrib/compiler-rt/lib/lsan/lsan_allocator.h index 5a0d94c71415..4006f7929269 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_allocator.h +++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.h @@ -68,9 +68,16 @@ struct AP32 { }; typedef SizeClassAllocator32<AP32> PrimaryAllocator; #elif defined(__x86_64__) || defined(__powerpc64__) +# if defined(__powerpc64__) +const uptr kAllocatorSpace = 0xa0000000000ULL; +const uptr kAllocatorSize = 0x20000000000ULL; // 2T. +# else +const uptr kAllocatorSpace = 0x600000000000ULL; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +# endif struct AP64 { // Allocator64 parameters. Deliberately using a short name. - static const uptr kSpaceBeg = 0x600000000000ULL; - static const uptr kSpaceSize = 0x40000000000ULL; // 4T. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = kAllocatorSize; static const uptr kMetadataSize = sizeof(ChunkMetadata); typedef DefaultSizeClassMap SizeClassMap; typedef NoOpMapUnmapCallback MapUnmapCallback; diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.cc b/contrib/compiler-rt/lib/lsan/lsan_common.cc index c121e6a8fb24..69ffda539a26 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common.cc @@ -57,12 +57,12 @@ void RegisterLsanFlags(FlagParser *parser, Flags *f) { #define LOG_POINTERS(...) \ do { \ if (flags()->log_pointers) Report(__VA_ARGS__); \ - } while (0); + } while (0) #define LOG_THREADS(...) \ do { \ if (flags()->log_threads) Report(__VA_ARGS__); \ - } while (0); + } while (0) ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; static SuppressionContext *suppression_ctx = nullptr; @@ -107,6 +107,10 @@ void InitializeRootRegions() { root_regions = new(placeholder) InternalMmapVector<RootRegion>(1); } +const char *MaybeCallLsanDefaultOptions() { + return (&__lsan_default_options) ? __lsan_default_options() : ""; +} + void InitCommonLsan() { InitializeRootRegions(); if (common_flags()->detect_leaks) { @@ -122,7 +126,6 @@ class Decorator: public __sanitizer::SanitizerCommonDecorator { Decorator() : SanitizerCommonDecorator() { } const char *Error() { return Red(); } const char *Leak() { return Blue(); } - const char *End() { return Default(); } }; static inline bool CanBeAHeapPointer(uptr p) { @@ -408,8 +411,9 @@ static void MarkInvalidPCCb(uptr chunk, void *arg) { } } -// On Linux, handles dynamically allocated TLS blocks by treating all chunks -// allocated from ld-linux.so as reachable. +// On Linux, treats all chunks allocated from ld-linux.so as reachable, which +// covers dynamically allocated TLS blocks, internal dynamic loader's loaded +// modules accounting etc. // Dynamic TLS blocks contain the TLS variables of dynamically loaded modules. // They are allocated with a __libc_memalign() call in allocate_and_init() // (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those @@ -564,7 +568,7 @@ static bool CheckForLeaks() { "\n"); Printf("%s", d.Error()); Report("ERROR: LeakSanitizer: detected memory leaks\n"); - Printf("%s", d.End()); + Printf("%s", d.Default()); param.leak_report.ReportTopLeaks(flags()->max_leaks); } if (common_flags()->print_suppressions) @@ -594,6 +598,8 @@ static int DoRecoverableLeakCheck() { return have_leaks ? 1 : 0; } +void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); } + static Suppression *GetSuppressionForAddr(uptr addr) { Suppression *s = nullptr; @@ -698,7 +704,7 @@ void LeakReport::PrintReportForLeak(uptr index) { Printf("%s leak of %zu byte(s) in %zu object(s) allocated from:\n", leaks_[index].is_directly_leaked ? "Direct" : "Indirect", leaks_[index].total_size, leaks_[index].hit_count); - Printf("%s", d.End()); + Printf("%s", d.Default()); PrintStackTraceById(leaks_[index].stack_trace_id); @@ -756,6 +762,7 @@ uptr LeakReport::UnsuppressedLeakCount() { namespace __lsan { void InitCommonLsan() { } void DoLeakCheck() { } +void DoRecoverableLeakCheckVoid() { } void DisableInThisThread() { } void EnableInThisThread() { } } @@ -854,6 +861,11 @@ int __lsan_do_recoverable_leak_check() { #if !SANITIZER_SUPPORTS_WEAK_HOOKS SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char * __lsan_default_options() { + return ""; +} + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int __lsan_is_turned_off() { return 0; } diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.h b/contrib/compiler-rt/lib/lsan/lsan_common.h index 31bf3eb1df42..f3863309d893 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.h +++ b/contrib/compiler-rt/lib/lsan/lsan_common.h @@ -27,9 +27,9 @@ // because of "small" (4 bytes) pointer size that leads to high false negative // ratio on large leaks. But we still want to have it for some 32 bit arches // (e.g. x86), see https://github.com/google/sanitizers/issues/403. -// To enable LeakSanitizer on new architecture, one need to implement -// internal_clone function as well as (probably) adjust TLS machinery for -// new architecture inside sanitizer library. +// To enable LeakSanitizer on a new architecture, one needs to implement the +// internal_clone function as well as (probably) adjust the TLS machinery for +// the new architecture inside the sanitizer library. #if (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC) && \ (SANITIZER_WORDSIZE == 64) && \ (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \ @@ -143,8 +143,10 @@ enum IgnoreObjectResult { }; // Functions called from the parent tool. +const char *MaybeCallLsanDefaultOptions(); void InitCommonLsan(); void DoLeakCheck(); +void DoRecoverableLeakCheckVoid(); void DisableCounterUnderflow(); bool DisabledInThisThread(); @@ -250,6 +252,9 @@ class LsanMetadata { extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char *__lsan_default_options(); + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int __lsan_is_turned_off(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc b/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc index 5042c7b3ada5..cdd7f032a192 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc @@ -20,6 +20,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_getauxval.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -30,8 +31,12 @@ static const char kLinkerName[] = "ld"; static char linker_placeholder[sizeof(LoadedModule)] ALIGNED(64); static LoadedModule *linker = nullptr; -static bool IsLinker(const char* full_name) { - return LibraryNameIs(full_name, kLinkerName); +static bool IsLinker(const LoadedModule& module) { +#if SANITIZER_USE_GETAUXVAL + return module.base_address() == getauxval(AT_BASE); +#else + return LibraryNameIs(module.full_name(), kLinkerName); +#endif // SANITIZER_USE_GETAUXVAL } __attribute__((tls_model("initial-exec"))) @@ -49,22 +54,25 @@ void InitializePlatformSpecificModules() { ListOfModules modules; modules.init(); for (LoadedModule &module : modules) { - if (!IsLinker(module.full_name())) continue; + if (!IsLinker(module)) + continue; if (linker == nullptr) { linker = reinterpret_cast<LoadedModule *>(linker_placeholder); *linker = module; module = LoadedModule(); } else { VReport(1, "LeakSanitizer: Multiple modules match \"%s\". " - "TLS will not be handled correctly.\n", kLinkerName); + "TLS and other allocations originating from linker might be " + "falsely reported as leaks.\n", kLinkerName); linker->clear(); linker = nullptr; return; } } if (linker == nullptr) { - VReport(1, "LeakSanitizer: Dynamic linker not found. " - "TLS will not be handled correctly.\n"); + VReport(1, "LeakSanitizer: Dynamic linker not found. TLS and other " + "allocations originating from linker might be falsely reported " + "as leaks.\n"); } } diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc index ade94340ae81..ac27c7af6e35 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc @@ -92,8 +92,25 @@ LoadedModule *GetLinker() { return nullptr; } // required on Darwin. void InitializePlatformSpecificModules() {} +// Sections which can't contain contain global pointers. This list errs on the +// side of caution to avoid false positives, at the expense of performance. +// +// Other potentially safe sections include: +// __all_image_info, __crash_info, __const, __got, __interpose, __objc_msg_break +// +// Sections which definitely cannot be included here are: +// __objc_data, __objc_const, __data, __bss, __common, __thread_data, +// __thread_bss, __thread_vars, __objc_opt_rw, __objc_opt_ptrs +static const char *kSkippedSecNames[] = { + "__cfstring", "__la_symbol_ptr", "__mod_init_func", + "__mod_term_func", "__nl_symbol_ptr", "__objc_classlist", + "__objc_classrefs", "__objc_imageinfo", "__objc_nlclslist", + "__objc_protolist", "__objc_selrefs", "__objc_superrefs"}; + // Scans global variables for heap pointers. void ProcessGlobalRegions(Frontier *frontier) { + for (auto name : kSkippedSecNames) CHECK(ARRAY_SIZE(name) < kMaxSegName); + MemoryMappingLayout memory_mapping(false); InternalMmapVector<LoadedModule> modules(/*initial_capacity*/ 128); memory_mapping.DumpListOfModules(&modules); @@ -107,6 +124,10 @@ void ProcessGlobalRegions(Frontier *frontier) { // Sections storing global variables are writable and non-executable if (range.executable || !range.writable) continue; + for (auto name : kSkippedSecNames) { + if (!internal_strcmp(range.name, name)) continue; + } + ScanGlobalRange(range.beg, range.end, frontier); } } diff --git a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc index 168868b012bc..b3e73e3896d5 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc @@ -20,6 +20,7 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_interceptors.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" @@ -44,6 +45,7 @@ int pthread_setspecific(unsigned key, const void *v); namespace std { struct nothrow_t; + enum class align_val_t: size_t; } #if !SANITIZER_MAC @@ -203,13 +205,19 @@ INTERCEPTOR(int, mprobe, void *ptr) { #define OPERATOR_NEW_BODY(nothrow) \ ENSURE_LSAN_INITED; \ GET_STACK_TRACE_MALLOC; \ - void *res = Allocate(stack, size, 1, kAlwaysClearMemory);\ - if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\ + void *res = lsan_malloc(size, stack); \ + if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM(); \ + return res; +#define OPERATOR_NEW_BODY_ALIGN(nothrow) \ + ENSURE_LSAN_INITED; \ + GET_STACK_TRACE_MALLOC; \ + void *res = lsan_memalign((uptr)align, size, stack); \ + if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM(); \ return res; #define OPERATOR_DELETE_BODY \ ENSURE_LSAN_INITED; \ - Deallocate(ptr); + lsan_free(ptr); // On OS X it's not enough to just provide our own 'operator new' and // 'operator delete' implementations, because they're going to be in the runtime @@ -229,6 +237,18 @@ void *operator new(size_t size, std::nothrow_t const&) INTERCEPTOR_ATTRIBUTE void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } @@ -239,6 +259,30 @@ void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE void operator delete[](void *ptr, std::nothrow_t const &) { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, size_t size, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } #else // SANITIZER_MAC @@ -265,6 +309,7 @@ INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) ///// Thread initialization and finalization. ///// +#if !SANITIZER_NETBSD static unsigned g_thread_finalize_key; static void thread_finalize(void *v) { @@ -278,6 +323,18 @@ static void thread_finalize(void *v) { } ThreadFinish(); } +#endif + +#if SANITIZER_NETBSD +INTERCEPTOR(void, _lwp_exit) { + ENSURE_LSAN_INITED; + ThreadFinish(); + REAL(_lwp_exit)(); +} +#define LSAN_MAYBE_INTERCEPT__LWP_EXIT INTERCEPT_FUNCTION(_lwp_exit) +#else +#define LSAN_MAYBE_INTERCEPT__LWP_EXIT +#endif struct ThreadParam { void *(*callback)(void *arg); @@ -291,11 +348,13 @@ extern "C" void *__lsan_thread_start_func(void *arg) { void *param = p->param; // Wait until the last iteration to maximize the chance that we are the last // destructor to run. +#if !SANITIZER_NETBSD if (pthread_setspecific(g_thread_finalize_key, (void*)GetPthreadDestructorIterations())) { Report("LeakSanitizer: failed to set thread key.\n"); Die(); } +#endif int tid = 0; while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) internal_sched_yield(); @@ -357,9 +416,14 @@ INTERCEPTOR(void, _exit, int status) { REAL(_exit)(status); } +#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + namespace __lsan { void InitializeInterceptors() { + InitializeSignalInterceptors(); + INTERCEPT_FUNCTION(malloc); INTERCEPT_FUNCTION(free); LSAN_MAYBE_INTERCEPT_CFREE; @@ -378,10 +442,14 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(_exit); + LSAN_MAYBE_INTERCEPT__LWP_EXIT; + +#if !SANITIZER_NETBSD if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { Report("LeakSanitizer: failed to create thread key.\n"); Die(); } +#endif } } // namespace __lsan diff --git a/contrib/compiler-rt/lib/lsan/weak_symbols.txt b/contrib/compiler-rt/lib/lsan/weak_symbols.txt index da4f994da865..692255679bee 100644 --- a/contrib/compiler-rt/lib/lsan/weak_symbols.txt +++ b/contrib/compiler-rt/lib/lsan/weak_symbols.txt @@ -1,2 +1,3 @@ +___lsan_default_options ___lsan_default_suppressions ___lsan_is_turned_off diff --git a/contrib/compiler-rt/lib/msan/msan.cc b/contrib/compiler-rt/lib/msan/msan.cc index d2981f0b0edb..e6226ba7670f 100644 --- a/contrib/compiler-rt/lib/msan/msan.cc +++ b/contrib/compiler-rt/lib/msan/msan.cc @@ -218,14 +218,14 @@ static void InitializeFlags() { } void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - bool request_fast_unwind) { + void *context, bool request_fast_unwind) { MsanThread *t = GetCurrentThread(); if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { // Block reports from our interceptors during _Unwind_Backtrace. SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, nullptr, 0, 0, request_fast_unwind); + return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); } - stack->Unwind(max_s, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), + stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), request_fast_unwind); } @@ -369,6 +369,16 @@ void __msan_warning_noreturn() { Die(); } +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +static void MsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr); +} + void __msan_init() { CHECK(!msan_init_is_running); if (msan_inited) return; @@ -384,6 +394,7 @@ void __msan_init() { __sanitizer_set_report_path(common_flags()->log_path); InitializeInterceptors(); + InstallDeadlySignalHandlers(MsanOnDeadlySignal); InstallAtExitHandler(); // Needs __cxa_atexit interceptor. DisableCoreDumperIfNecessary(); diff --git a/contrib/compiler-rt/lib/msan/msan.h b/contrib/compiler-rt/lib/msan/msan.h index fa9c15b88bef..cbae444127ee 100644 --- a/contrib/compiler-rt/lib/msan/msan.h +++ b/contrib/compiler-rt/lib/msan/msan.h @@ -160,21 +160,25 @@ const MappingDesc kMemoryLayout[] = { # define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x1000000000ULL) #elif SANITIZER_LINUX && SANITIZER_PPC64 - const MappingDesc kMemoryLayout[] = { - {0x000000000000ULL, 0x000100000000ULL, MappingDesc::APP, "low memory"}, - {0x000100000000ULL, 0x080000000000ULL, MappingDesc::INVALID, "invalid"}, - {0x080000000000ULL, 0x180100000000ULL, MappingDesc::SHADOW, "shadow"}, - {0x180100000000ULL, 0x1C0000000000ULL, MappingDesc::INVALID, "invalid"}, - {0x1C0000000000ULL, 0x2C0100000000ULL, MappingDesc::ORIGIN, "origin"}, - {0x2C0100000000ULL, 0x300000000000ULL, MappingDesc::INVALID, "invalid"}, - {0x300000000000ULL, 0x400000000000ULL, MappingDesc::APP, "high memory"}}; + {0x000000000000ULL, 0x000200000000ULL, MappingDesc::APP, "low memory"}, + {0x000200000000ULL, 0x080000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x080000000000ULL, 0x180200000000ULL, MappingDesc::SHADOW, "shadow"}, + {0x180200000000ULL, 0x1C0000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x1C0000000000ULL, 0x2C0200000000ULL, MappingDesc::ORIGIN, "origin"}, + {0x2C0200000000ULL, 0x300000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x300000000000ULL, 0x800000000000ULL, MappingDesc::APP, "high memory"}}; +// Various kernels use different low end ranges but we can combine them into one +// big range. They also use different high end ranges but we can map them all to +// one range. // Maps low and high app ranges to contiguous space with zero base: -// Low: 0000 0000 0000 - 0000 ffff ffff -> 1000 0000 0000 - 1000 ffff ffff +// Low: 0000 0000 0000 - 0001 ffff ffff -> 1000 0000 0000 - 1001 ffff ffff // High: 3000 0000 0000 - 3fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff +// High: 4000 0000 0000 - 4fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff +// High: 7000 0000 0000 - 7fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff #define LINEARIZE_MEM(mem) \ - (((uptr)(mem) & ~0x200000000000ULL) ^ 0x100000000000ULL) + (((uptr)(mem) & ~0xE00000000000ULL) ^ 0x100000000000ULL) #define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x080000000000ULL) #define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x140000000000ULL) @@ -199,7 +203,7 @@ const MappingDesc kMemoryLayout[] = { #define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x100000000000ULL) #define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x280000000000) -#elif SANITIZER_LINUX && SANITIZER_WORDSIZE == 64 +#elif SANITIZER_NETBSD || (SANITIZER_LINUX && SANITIZER_WORDSIZE == 64) #ifdef MSAN_LINUX_X86_64_OLD_MAPPING // Requries PIE binary and ASLR enabled. @@ -310,7 +314,7 @@ void PrintWarning(uptr pc, uptr bp); void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin); void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - bool request_fast_unwind); + void *context, bool request_fast_unwind); void ReportUMR(StackTrace *stack, u32 origin); void ReportExpectedUMRNotFound(StackTrace *stack); @@ -330,32 +334,32 @@ u32 ChainOrigin(u32 id, StackTrace *stack); const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; -#define GET_MALLOC_STACK_TRACE \ - BufferedStackTrace stack; \ - if (__msan_get_track_origins() && msan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ +#define GET_MALLOC_STACK_TRACE \ + BufferedStackTrace stack; \ + if (__msan_get_track_origins() && msan_inited) \ + GetStackTrace(&stack, common_flags()->malloc_context_size, \ + StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ common_flags()->fast_unwind_on_malloc) // For platforms which support slow unwinder only, we restrict the store context // size to 1, basically only storing the current pc. We do this because the slow // unwinder which is based on libunwind is not async signal safe and causes // random freezes in forking applications as well as in signal handlers. -#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ - BufferedStackTrace stack; \ - if (__msan_get_track_origins() > 1 && msan_inited) { \ - if (!SANITIZER_CAN_FAST_UNWIND) \ - GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \ - false); \ - else \ - GetStackTrace(&stack, flags()->store_context_size, pc, bp, \ - common_flags()->fast_unwind_on_malloc); \ +#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (__msan_get_track_origins() > 1 && msan_inited) { \ + if (!SANITIZER_CAN_FAST_UNWIND) \ + GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \ + nullptr, false); \ + else \ + GetStackTrace(&stack, flags()->store_context_size, pc, bp, nullptr, \ + common_flags()->fast_unwind_on_malloc); \ } -#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ - BufferedStackTrace stack; \ - if (msan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, \ +#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (msan_inited) \ + GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ common_flags()->fast_unwind_on_fatal) #define GET_STORE_STACK_TRACE \ diff --git a/contrib/compiler-rt/lib/msan/msan_allocator.cc b/contrib/compiler-rt/lib/msan/msan_allocator.cc index 1034dbdf9b55..0f9942324931 100644 --- a/contrib/compiler-rt/lib/msan/msan_allocator.cc +++ b/contrib/compiler-rt/lib/msan/msan_allocator.cc @@ -62,7 +62,8 @@ struct MsanMapUnmapCallback { }; typedef SizeClassAllocator32<AP32> PrimaryAllocator; #elif defined(__x86_64__) -#if SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING) +#if SANITIZER_NETBSD || \ + (SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING)) static const uptr kAllocatorSpace = 0x700000000000ULL; #else static const uptr kAllocatorSpace = 0x600000000000ULL; @@ -255,8 +256,12 @@ void *msan_valloc(uptr size, StackTrace *stack) { void *msan_pvalloc(uptr size, StackTrace *stack) { uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + return Allocator::FailureHandler::OnBadRequest(); + } // pvalloc(0) should allocate one page. - size = size == 0 ? PageSize : RoundUpTo(size, PageSize); + size = size ? RoundUpTo(size, PageSize) : PageSize; return SetErrnoOnNull(MsanAllocate(stack, size, PageSize, false)); } diff --git a/contrib/compiler-rt/lib/msan/msan_interceptors.cc b/contrib/compiler-rt/lib/msan/msan_interceptors.cc index b5d22baca08d..a7fe09b25ffb 100644 --- a/contrib/compiler-rt/lib/msan/msan_interceptors.cc +++ b/contrib/compiler-rt/lib/msan/msan_interceptors.cc @@ -22,6 +22,7 @@ #include "msan_thread.h" #include "msan_poisoning.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_allocator.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_allocator_internal.h" @@ -33,6 +34,11 @@ #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" +#if SANITIZER_NETBSD +#define gettimeofday __gettimeofday50 +#define getrusage __getrusage50 +#endif + #include <stdarg.h> // ACHTUNG! No other system header includes in this file. // Ideally, we should get rid of stdarg.h as well. @@ -86,22 +92,21 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { } while (0) // Check that [x, x+n) range is unpoisoned. -#define CHECK_UNPOISONED_0(x, n) \ - do { \ - sptr offset = __msan_test_shadow(x, n); \ - if (__msan::IsInSymbolizer()) \ - break; \ - if (offset >= 0 && __msan::flags()->report_umrs) { \ - GET_CALLER_PC_BP_SP; \ - (void) sp; \ - ReportUMRInsideAddressRange(__func__, x, n, offset); \ - __msan::PrintWarningWithOrigin( \ - pc, bp, __msan_get_origin((const char *)x + offset)); \ - if (__msan::flags()->halt_on_error) { \ - Printf("Exiting\n"); \ - Die(); \ - } \ - } \ +#define CHECK_UNPOISONED_0(x, n) \ + do { \ + sptr __offset = __msan_test_shadow(x, n); \ + if (__msan::IsInSymbolizer()) break; \ + if (__offset >= 0 && __msan::flags()->report_umrs) { \ + GET_CALLER_PC_BP_SP; \ + (void)sp; \ + ReportUMRInsideAddressRange(__func__, x, n, __offset); \ + __msan::PrintWarningWithOrigin( \ + pc, bp, __msan_get_origin((const char *)x + __offset)); \ + if (__msan::flags()->halt_on_error) { \ + Printf("Exiting\n"); \ + Die(); \ + } \ + } \ } while (0) // Check that [x, x+n) range is unpoisoned unless we are in a nested @@ -109,7 +114,7 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { #define CHECK_UNPOISONED(x, n) \ do { \ if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \ - } while (0); + } while (0) #define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \ CHECK_UNPOISONED((x), \ @@ -118,7 +123,7 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { #define CHECK_UNPOISONED_STRING(x, n) \ CHECK_UNPOISONED_STRING_OF_LEN((x), internal_strlen(x), (n)) -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(SIZE_T, fread_unlocked, void *ptr, SIZE_T size, SIZE_T nmemb, void *file) { ENSURE_MSAN_INITED(); @@ -134,16 +139,21 @@ INTERCEPTOR(SIZE_T, fread_unlocked, void *ptr, SIZE_T size, SIZE_T nmemb, INTERCEPTOR(SSIZE_T, readlink, const char *path, char *buf, SIZE_T bufsiz) { ENSURE_MSAN_INITED(); - CHECK_UNPOISONED_STRING(path, 0) + CHECK_UNPOISONED_STRING(path, 0); SSIZE_T res = REAL(readlink)(path, buf, bufsiz); if (res > 0) __msan_unpoison(buf, res); return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(void *, mempcpy, void *dest, const void *src, SIZE_T n) { return (char *)__msan_memcpy(dest, src, n) + n; } +#define MSAN_MAYBE_INTERCEPT_MEMPCPY INTERCEPT_FUNCTION(mempcpy) +#else +#define MSAN_MAYBE_INTERCEPT_MEMPCPY +#endif INTERCEPTOR(void *, memccpy, void *dest, const void *src, int c, SIZE_T n) { ENSURE_MSAN_INITED(); @@ -168,7 +178,7 @@ INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) { GET_MALLOC_STACK_TRACE; return msan_memalign(alignment, size, &stack); @@ -183,6 +193,7 @@ INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) { return msan_aligned_alloc(alignment, size, &stack); } +#if !SANITIZER_NETBSD INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { GET_MALLOC_STACK_TRACE; void *ptr = msan_memalign(alignment, size, &stack); @@ -190,13 +201,17 @@ INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { DTLS_on_libc_memalign(ptr, size); return ptr; } +#define MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN INTERCEPT_FUNCTION(__libc_memalign) +#else +#define MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN +#endif INTERCEPTOR(void *, valloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; return msan_valloc(size, &stack); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void *, pvalloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; return msan_pvalloc(size, &stack); @@ -212,7 +227,7 @@ INTERCEPTOR(void, free, void *ptr) { MsanDeallocate(&stack, ptr); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void, cfree, void *ptr) { GET_MALLOC_STACK_TRACE; if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; @@ -223,11 +238,17 @@ INTERCEPTOR(void, cfree, void *ptr) { #define MSAN_MAYBE_INTERCEPT_CFREE #endif +#if !SANITIZER_NETBSD INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { return __sanitizer_get_allocated_size(ptr); } +#define MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE \ + INTERCEPT_FUNCTION(malloc_usable_size) +#else +#define MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE +#endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD // This function actually returns a struct by value, but we can't unpoison a // temporary! The following is equivalent on all supported platforms but // aarch64 (which uses a different register for sret value). We have a test @@ -246,7 +267,7 @@ INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { #define MSAN_MAYBE_INTERCEPT_MALLINFO #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, mallopt, int cmd, int value) { return -1; } @@ -255,7 +276,7 @@ INTERCEPTOR(int, mallopt, int cmd, int value) { #define MSAN_MAYBE_INTERCEPT_MALLOPT #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void, malloc_stats, void) { // FIXME: implement, but don't call REAL(malloc_stats)! } @@ -286,6 +307,7 @@ INTERCEPTOR(char *, strncpy, char *dest, const char *src, SIZE_T n) { // NOLINT return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(char *, stpcpy, char *dest, const char *src) { // NOLINT ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; @@ -295,6 +317,10 @@ INTERCEPTOR(char *, stpcpy, char *dest, const char *src) { // NOLINT CopyShadowAndOrigin(dest, src, n + 1, &stack); return res; } +#define MSAN_MAYBE_INTERCEPT_STPCPY INTERCEPT_FUNCTION(stpcpy) +#else +#define MSAN_MAYBE_INTERCEPT_STPCPY +#endif INTERCEPTOR(char *, strdup, char *src) { ENSURE_MSAN_INITED(); @@ -308,7 +334,7 @@ INTERCEPTOR(char *, strdup, char *src) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(char *, __strdup, char *src) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; @@ -323,6 +349,7 @@ INTERCEPTOR(char *, __strdup, char *src) { #define MSAN_MAYBE_INTERCEPT___STRDUP #endif +#if !SANITIZER_NETBSD INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) { ENSURE_MSAN_INITED(); char *res = REAL(gcvt)(number, ndigit, buf); @@ -330,6 +357,10 @@ INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) { __msan_unpoison(buf, n + 1); return res; } +#define MSAN_MAYBE_INTERCEPT_GCVT INTERCEPT_FUNCTION(gcvt) +#else +#define MSAN_MAYBE_INTERCEPT_GCVT +#endif INTERCEPTOR(char *, strcat, char *dest, const char *src) { // NOLINT ENSURE_MSAN_INITED(); @@ -386,6 +417,16 @@ INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) { // NOLINT INTERCEPTOR_STRTO_BODY(ret_type, func, nptr, endptr, base, loc); \ } +#if SANITIZER_NETBSD +#define INTERCEPTORS_STRTO(ret_type, func, char_type) \ + INTERCEPTOR_STRTO(ret_type, func, char_type) \ + INTERCEPTOR_STRTO_LOC(ret_type, func##_l, char_type) + +#define INTERCEPTORS_STRTO_BASE(ret_type, func, char_type) \ + INTERCEPTOR_STRTO_BASE(ret_type, func, char_type) \ + INTERCEPTOR_STRTO_BASE_LOC(ret_type, func##_l, char_type) + +#else #define INTERCEPTORS_STRTO(ret_type, func, char_type) \ INTERCEPTOR_STRTO(ret_type, func, char_type) \ INTERCEPTOR_STRTO_LOC(ret_type, func##_l, char_type) \ @@ -397,6 +438,7 @@ INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) { // NOLINT INTERCEPTOR_STRTO_BASE_LOC(ret_type, func##_l, char_type) \ INTERCEPTOR_STRTO_BASE_LOC(ret_type, __##func##_l, char_type) \ INTERCEPTOR_STRTO_BASE_LOC(ret_type, __##func##_internal, char_type) +#endif INTERCEPTORS_STRTO(double, strtod, char) // NOLINT INTERCEPTORS_STRTO(float, strtof, char) // NOLINT @@ -405,6 +447,7 @@ INTERCEPTORS_STRTO_BASE(long, strtol, char) // NOLINT INTERCEPTORS_STRTO_BASE(long long, strtoll, char) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long, strtoul, char) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long long, strtoull, char) // NOLINT +INTERCEPTORS_STRTO_BASE(u64, strtouq, char) // NOLINT INTERCEPTORS_STRTO(double, wcstod, wchar_t) // NOLINT INTERCEPTORS_STRTO(float, wcstof, wchar_t) // NOLINT @@ -414,11 +457,17 @@ INTERCEPTORS_STRTO_BASE(long long, wcstoll, wchar_t) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long, wcstoul, wchar_t) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long long, wcstoull, wchar_t) // NOLINT +#if SANITIZER_NETBSD +#define INTERCEPT_STRTO(func) \ + INTERCEPT_FUNCTION(func); \ + INTERCEPT_FUNCTION(func##_l); +#else #define INTERCEPT_STRTO(func) \ INTERCEPT_FUNCTION(func); \ INTERCEPT_FUNCTION(func##_l); \ INTERCEPT_FUNCTION(__##func##_l); \ INTERCEPT_FUNCTION(__##func##_internal); +#endif // FIXME: support *wprintf in common format interceptors. @@ -457,6 +506,20 @@ INTERCEPTOR(SIZE_T, strxfrm_l, char *dest, const char *src, SIZE_T n, return res; } +#if SANITIZER_LINUX +INTERCEPTOR(SIZE_T, __strxfrm_l, char *dest, const char *src, SIZE_T n, + void *loc) { + ENSURE_MSAN_INITED(); + CHECK_UNPOISONED(src, REAL(strlen)(src) + 1); + SIZE_T res = REAL(__strxfrm_l)(dest, src, n, loc); + if (res < n) __msan_unpoison(dest, res + 1); + return res; +} +#define MSAN_MAYBE_INTERCEPT___STRXFRM_L INTERCEPT_FUNCTION(__strxfrm_l) +#else +#define MSAN_MAYBE_INTERCEPT___STRXFRM_L +#endif + #define INTERCEPTOR_STRFTIME_BODY(char_type, ret_type, func, s, ...) \ ENSURE_MSAN_INITED(); \ ret_type res = REAL(func)(s, __VA_ARGS__); \ @@ -473,7 +536,7 @@ INTERCEPTOR(SIZE_T, strftime_l, char *s, SIZE_T max, const char *format, INTERCEPTOR_STRFTIME_BODY(char, SIZE_T, strftime_l, s, max, format, tm, loc); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(SIZE_T, __strftime_l, char *s, SIZE_T max, const char *format, __sanitizer_tm *tm, void *loc) { INTERCEPTOR_STRFTIME_BODY(char, SIZE_T, __strftime_l, s, max, format, tm, @@ -495,7 +558,7 @@ INTERCEPTOR(SIZE_T, wcsftime_l, wchar_t *s, SIZE_T max, const wchar_t *format, loc); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(SIZE_T, __wcsftime_l, wchar_t *s, SIZE_T max, const wchar_t *format, __sanitizer_tm *tm, void *loc) { INTERCEPTOR_STRFTIME_BODY(wchar_t, SIZE_T, __wcsftime_l, s, max, format, tm, @@ -513,7 +576,8 @@ INTERCEPTOR(int, mbtowc, wchar_t *dest, const char *src, SIZE_T n) { return res; } -INTERCEPTOR(int, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, void *ps) { +INTERCEPTOR(SIZE_T, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, + void *ps) { ENSURE_MSAN_INITED(); SIZE_T res = REAL(mbrtowc)(dest, src, n, ps); if (res != (SIZE_T)-1 && dest) __msan_unpoison(dest, sizeof(wchar_t)); @@ -529,6 +593,7 @@ INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; @@ -536,6 +601,10 @@ INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { CopyShadowAndOrigin(dest, src, n * sizeof(wchar_t), &stack); return res; } +#define MSAN_MAYBE_INTERCEPT_WMEMPCPY INTERCEPT_FUNCTION(wmempcpy) +#else +#define MSAN_MAYBE_INTERCEPT_WMEMPCPY +#endif INTERCEPTOR(wchar_t *, wmemset, wchar_t *s, wchar_t c, SIZE_T n) { CHECK(MEM_IS_APP(s)); @@ -569,6 +638,7 @@ INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(char *, fcvt, double x, int a, int *b, int *c) { ENSURE_MSAN_INITED(); char *res = REAL(fcvt)(x, a, b, c); @@ -577,6 +647,10 @@ INTERCEPTOR(char *, fcvt, double x, int a, int *b, int *c) { if (res) __msan_unpoison(res, REAL(strlen)(res) + 1); return res; } +#define MSAN_MAYBE_INTERCEPT_FCVT INTERCEPT_FUNCTION(fcvt) +#else +#define MSAN_MAYBE_INTERCEPT_FCVT +#endif INTERCEPTOR(char *, getenv, char *name) { if (msan_init_is_running) @@ -601,7 +675,7 @@ static void UnpoisonEnviron() { INTERCEPTOR(int, setenv, const char *name, const char *value, int overwrite) { ENSURE_MSAN_INITED(); - CHECK_UNPOISONED_STRING(name, 0) + CHECK_UNPOISONED_STRING(name, 0); int res = REAL(setenv)(name, value, overwrite); if (!res) UnpoisonEnviron(); return res; @@ -614,7 +688,7 @@ INTERCEPTOR(int, putenv, char *string) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) { ENSURE_MSAN_INITED(); int res = REAL(__fxstat)(magic, fd, buf); @@ -627,7 +701,7 @@ INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) { #define MSAN_MAYBE_INTERCEPT___FXSTAT #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) { ENSURE_MSAN_INITED(); int res = REAL(__fxstat64)(magic, fd, buf); @@ -640,7 +714,7 @@ INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) { #define MSAN_MAYBE_INTERCEPT___FXSTAT64 #endif -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD INTERCEPTOR(int, fstatat, int fd, char *pathname, void *buf, int flags) { ENSURE_MSAN_INITED(); int res = REAL(fstatat)(fd, pathname, buf, flags); @@ -659,7 +733,7 @@ INTERCEPTOR(int, __fxstatat, int magic, int fd, char *pathname, void *buf, # define MSAN_INTERCEPT_FSTATAT INTERCEPT_FUNCTION(__fxstatat) #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, __fxstatat64, int magic, int fd, char *pathname, void *buf, int flags) { ENSURE_MSAN_INITED(); @@ -706,7 +780,7 @@ INTERCEPTOR(char *, fgets, char *s, int size, void *stream) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(char *, fgets_unlocked, char *s, int size, void *stream) { ENSURE_MSAN_INITED(); char *res = REAL(fgets_unlocked)(s, size, stream); @@ -729,7 +803,7 @@ INTERCEPTOR(int, getrlimit, int resource, void *rlim) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, getrlimit64, int resource, void *rlim) { if (msan_init_is_running) return REAL(getrlimit64)(resource, rlim); ENSURE_MSAN_INITED(); @@ -805,7 +879,7 @@ INTERCEPTOR(int, gethostname, char *name, SIZE_T len) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, epoll_wait, int epfd, void *events, int maxevents, int timeout) { ENSURE_MSAN_INITED(); @@ -820,7 +894,7 @@ INTERCEPTOR(int, epoll_wait, int epfd, void *events, int maxevents, #define MSAN_MAYBE_INTERCEPT_EPOLL_WAIT #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, epoll_pwait, int epfd, void *events, int maxevents, int timeout, void *sigmask) { ENSURE_MSAN_INITED(); @@ -909,7 +983,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags, return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags, int fd, OFF64_T offset) { ENSURE_MSAN_INITED(); @@ -983,59 +1057,13 @@ static void SignalAction(int signo, void *si, void *uc) { cb(signo, si, uc); } -INTERCEPTOR(int, sigaction, int signo, const __sanitizer_sigaction *act, - __sanitizer_sigaction *oldact) { - ENSURE_MSAN_INITED(); - // FIXME: check that *act is unpoisoned. - // That requires intercepting all of sigemptyset, sigfillset, etc. - int res; - if (flags()->wrap_signals) { - SpinMutexLock lock(&sigactions_mu); - CHECK_LT(signo, kMaxSignals); - uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed); - __sanitizer_sigaction new_act; - __sanitizer_sigaction *pnew_act = act ? &new_act : nullptr; - if (act) { - REAL(memcpy)(pnew_act, act, sizeof(__sanitizer_sigaction)); - uptr cb = (uptr)pnew_act->sigaction; - uptr new_cb = (pnew_act->sa_flags & __sanitizer::sa_siginfo) - ? (uptr)SignalAction - : (uptr)SignalHandler; - if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { - atomic_store(&sigactions[signo], cb, memory_order_relaxed); - pnew_act->sigaction = (void (*)(int, void *, void *))new_cb; - } - } - res = REAL(sigaction)(signo, pnew_act, oldact); - if (res == 0 && oldact) { - uptr cb = (uptr)oldact->sigaction; - if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { - oldact->sigaction = (void (*)(int, void *, void *))old_cb; - } - } - } else { - res = REAL(sigaction)(signo, act, oldact); - } - - if (res == 0 && oldact) { - __msan_unpoison(oldact, sizeof(__sanitizer_sigaction)); - } - return res; -} - -INTERCEPTOR(int, signal, int signo, uptr cb) { - ENSURE_MSAN_INITED(); - if (flags()->wrap_signals) { - CHECK_LT(signo, kMaxSignals); - SpinMutexLock lock(&sigactions_mu); - if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { - atomic_store(&sigactions[signo], cb, memory_order_relaxed); - cb = (uptr) SignalHandler; - } - return REAL(signal)(signo, cb); - } else { - return REAL(signal)(signo, cb); - } +static void read_sigaction(const __sanitizer_sigaction *act) { + CHECK_UNPOISONED(&act->sa_flags, sizeof(act->sa_flags)); + if (act->sa_flags & __sanitizer::sa_siginfo) + CHECK_UNPOISONED(&act->sigaction, sizeof(act->sigaction)); + else + CHECK_UNPOISONED(&act->handler, sizeof(act->handler)); + CHECK_UNPOISONED(&act->sa_mask, sizeof(act->sa_mask)); } extern "C" int pthread_attr_init(void *attr); @@ -1080,6 +1108,11 @@ INTERCEPTOR(int, pthread_key_create, __sanitizer_pthread_key_t *key, return res; } +#if SANITIZER_NETBSD +INTERCEPTOR(void, __libc_thr_keycreate, void *m, void (*dtor)(void *value)) \ + ALIAS(WRAPPER_NAME(pthread_key_create)); +#endif + INTERCEPTOR(int, pthread_join, void *th, void **retval) { ENSURE_MSAN_INITED(); int res = REAL(pthread_join)(th, retval); @@ -1124,21 +1157,6 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, return REAL(__cxa_atexit)(MSanAtExitWrapper, r, dso_handle); } -DECLARE_REAL(int, shmctl, int shmid, int cmd, void *buf) - -INTERCEPTOR(void *, shmat, int shmid, const void *shmaddr, int shmflg) { - ENSURE_MSAN_INITED(); - void *p = REAL(shmat)(shmid, shmaddr, shmflg); - if (p != (void *)-1) { - __sanitizer_shmid_ds ds; - int res = REAL(shmctl)(shmid, shmctl_ipc_stat, &ds); - if (!res) { - __msan_unpoison(p, ds.shm_segsz); - } - } - return p; -} - static void BeforeFork() { StackDepotLockAll(); ChainedOriginDepotLockAll(); @@ -1293,6 +1311,73 @@ int OnExit() { #include "sanitizer_common/sanitizer_platform_interceptors.h" #include "sanitizer_common/sanitizer_common_interceptors.inc" +static uptr signal_impl(int signo, uptr cb); +static int sigaction_impl(int signo, const __sanitizer_sigaction *act, + __sanitizer_sigaction *oldact); + +#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ + { return sigaction_impl(signo, act, oldact); } + +#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ + { \ + handler = signal_impl(signo, handler); \ + return REAL(func)(signo, handler); \ + } + +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +static int sigaction_impl(int signo, const __sanitizer_sigaction *act, + __sanitizer_sigaction *oldact) { + ENSURE_MSAN_INITED(); + if (act) read_sigaction(act); + int res; + if (flags()->wrap_signals) { + SpinMutexLock lock(&sigactions_mu); + CHECK_LT(signo, kMaxSignals); + uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed); + __sanitizer_sigaction new_act; + __sanitizer_sigaction *pnew_act = act ? &new_act : nullptr; + if (act) { + REAL(memcpy)(pnew_act, act, sizeof(__sanitizer_sigaction)); + uptr cb = (uptr)pnew_act->sigaction; + uptr new_cb = (pnew_act->sa_flags & __sanitizer::sa_siginfo) + ? (uptr)SignalAction + : (uptr)SignalHandler; + if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { + atomic_store(&sigactions[signo], cb, memory_order_relaxed); + pnew_act->sigaction = (decltype(pnew_act->sigaction))new_cb; + } + } + res = REAL(SIGACTION_SYMNAME)(signo, pnew_act, oldact); + if (res == 0 && oldact) { + uptr cb = (uptr)oldact->sigaction; + if (cb == (uptr)SignalAction || cb == (uptr)SignalHandler) { + oldact->sigaction = (decltype(oldact->sigaction))old_cb; + } + } + } else { + res = REAL(SIGACTION_SYMNAME)(signo, act, oldact); + } + + if (res == 0 && oldact) { + __msan_unpoison(oldact, sizeof(__sanitizer_sigaction)); + } + return res; +} + +static uptr signal_impl(int signo, uptr cb) { + ENSURE_MSAN_INITED(); + if (flags()->wrap_signals) { + CHECK_LT(signo, kMaxSignals); + SpinMutexLock lock(&sigactions_mu); + if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { + atomic_store(&sigactions[signo], cb, memory_order_relaxed); + cb = (uptr)&SignalHandler; + } + } + return cb; +} + #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s) #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ do { \ @@ -1353,6 +1438,19 @@ static int msan_dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, return cbdata->callback(info, size, cbdata->data); } +INTERCEPTOR(void *, shmat, int shmid, const void *shmaddr, int shmflg) { + ENSURE_MSAN_INITED(); + void *p = REAL(shmat)(shmid, shmaddr, shmflg); + if (p != (void *)-1) { + __sanitizer_shmid_ds ds; + int res = REAL(shmctl)(shmid, shmctl_ipc_stat, &ds); + if (!res) { + __msan_unpoison(p, ds.shm_segsz); + } + } + return p; +} + INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb callback, void *data) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, dl_iterate_phdr, callback, data); @@ -1455,12 +1553,13 @@ void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); InitializeCommonInterceptors(); + InitializeSignalInterceptors(); INTERCEPT_FUNCTION(mmap); MSAN_MAYBE_INTERCEPT_MMAP64; INTERCEPT_FUNCTION(posix_memalign); MSAN_MAYBE_INTERCEPT_MEMALIGN; - INTERCEPT_FUNCTION(__libc_memalign); + MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN; INTERCEPT_FUNCTION(valloc); MSAN_MAYBE_INTERCEPT_PVALLOC; INTERCEPT_FUNCTION(malloc); @@ -1468,7 +1567,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); MSAN_MAYBE_INTERCEPT_CFREE; - INTERCEPT_FUNCTION(malloc_usable_size); + MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE; MSAN_MAYBE_INTERCEPT_MALLINFO; MSAN_MAYBE_INTERCEPT_MALLOPT; MSAN_MAYBE_INTERCEPT_MALLOC_STATS; @@ -1476,18 +1575,18 @@ void InitializeInterceptors() { MSAN_MAYBE_INTERCEPT_FREAD_UNLOCKED; INTERCEPT_FUNCTION(readlink); INTERCEPT_FUNCTION(memccpy); - INTERCEPT_FUNCTION(mempcpy); + MSAN_MAYBE_INTERCEPT_MEMPCPY; INTERCEPT_FUNCTION(bcopy); INTERCEPT_FUNCTION(wmemset); INTERCEPT_FUNCTION(wmemcpy); - INTERCEPT_FUNCTION(wmempcpy); + MSAN_MAYBE_INTERCEPT_WMEMPCPY; INTERCEPT_FUNCTION(wmemmove); INTERCEPT_FUNCTION(strcpy); // NOLINT - INTERCEPT_FUNCTION(stpcpy); // NOLINT + MSAN_MAYBE_INTERCEPT_STPCPY; // NOLINT INTERCEPT_FUNCTION(strdup); MSAN_MAYBE_INTERCEPT___STRDUP; INTERCEPT_FUNCTION(strncpy); // NOLINT - INTERCEPT_FUNCTION(gcvt); + MSAN_MAYBE_INTERCEPT_GCVT; INTERCEPT_FUNCTION(strcat); // NOLINT INTERCEPT_FUNCTION(strncat); // NOLINT INTERCEPT_STRTO(strtod); @@ -1497,6 +1596,7 @@ void InitializeInterceptors() { INTERCEPT_STRTO(strtoul); INTERCEPT_STRTO(strtoll); INTERCEPT_STRTO(strtoull); + INTERCEPT_STRTO(strtouq); INTERCEPT_STRTO(wcstod); INTERCEPT_STRTO(wcstof); INTERCEPT_STRTO(wcstold); @@ -1513,6 +1613,7 @@ void InitializeInterceptors() { #endif INTERCEPT_FUNCTION(strxfrm); INTERCEPT_FUNCTION(strxfrm_l); + MSAN_MAYBE_INTERCEPT___STRXFRM_L; INTERCEPT_FUNCTION(strftime); INTERCEPT_FUNCTION(strftime_l); MSAN_MAYBE_INTERCEPT___STRFTIME_L; @@ -1531,7 +1632,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(setenv); INTERCEPT_FUNCTION(putenv); INTERCEPT_FUNCTION(gettimeofday); - INTERCEPT_FUNCTION(fcvt); + MSAN_MAYBE_INTERCEPT_FCVT; MSAN_MAYBE_INTERCEPT___FXSTAT; MSAN_INTERCEPT_FSTATAT; MSAN_MAYBE_INTERCEPT___FXSTAT64; @@ -1553,14 +1654,17 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(dlerror); INTERCEPT_FUNCTION(dl_iterate_phdr); INTERCEPT_FUNCTION(getrusage); - INTERCEPT_FUNCTION(sigaction); - INTERCEPT_FUNCTION(signal); #if defined(__mips__) INTERCEPT_FUNCTION_VER(pthread_create, "GLIBC_2.2"); #else INTERCEPT_FUNCTION(pthread_create); #endif INTERCEPT_FUNCTION(pthread_key_create); + +#if SANITIZER_NETBSD + INTERCEPT_FUNCTION(__libc_thr_keycreate); +#endif + INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(tzset); INTERCEPT_FUNCTION(__cxa_atexit); diff --git a/contrib/compiler-rt/lib/msan/msan_linux.cc b/contrib/compiler-rt/lib/msan/msan_linux.cc index 0a687f620c94..4e6321fcb918 100644 --- a/contrib/compiler-rt/lib/msan/msan_linux.cc +++ b/contrib/compiler-rt/lib/msan/msan_linux.cc @@ -9,11 +9,11 @@ // // This file is a part of MemorySanitizer. // -// Linux- and FreeBSD-specific code. +// Linux-, NetBSD- and FreeBSD-specific code. //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD #include "msan.h" #include "msan_thread.h" @@ -120,7 +120,7 @@ bool InitShadow(bool init_origins) { return false; } - const uptr maxVirtualAddress = GetMaxVirtualAddress(); + const uptr maxVirtualAddress = GetMaxUserVirtualAddress(); for (unsigned i = 0; i < kMemoryLayoutSize; ++i) { uptr start = kMemoryLayout[i].start; @@ -213,4 +213,4 @@ void MsanTSDDtor(void *tsd) { } // namespace __msan -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/msan/msan_report.cc b/contrib/compiler-rt/lib/msan/msan_report.cc index 9a35c9c13de5..28c9bbabb3e9 100644 --- a/contrib/compiler-rt/lib/msan/msan_report.cc +++ b/contrib/compiler-rt/lib/msan/msan_report.cc @@ -30,10 +30,8 @@ namespace __msan { class Decorator: public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() { } - const char *Warning() { return Red(); } const char *Origin() { return Magenta(); } const char *Name() { return Green(); } - const char *End() { return Default(); } }; static void DescribeStackOrigin(const char *so, uptr pc) { @@ -47,7 +45,7 @@ static void DescribeStackOrigin(const char *so, uptr pc) { " %sUninitialized value was created by an allocation of '%s%s%s'" " in the stack frame of function '%s%s%s'%s\n", d.Origin(), d.Name(), s, d.Origin(), d.Name(), sep + 1, d.Origin(), - d.End()); + d.Default()); InternalFree(s); if (pc) { @@ -66,7 +64,7 @@ static void DescribeOrigin(u32 id) { StackTrace stack; o = o.getNextChainedOrigin(&stack); Printf(" %sUninitialized value was stored to memory at%s\n", d.Origin(), - d.End()); + d.Default()); stack.Print(); } if (o.isStackOrigin()) { @@ -78,18 +76,19 @@ static void DescribeOrigin(u32 id) { switch (stack.tag) { case StackTrace::TAG_ALLOC: Printf(" %sUninitialized value was created by a heap allocation%s\n", - d.Origin(), d.End()); + d.Origin(), d.Default()); break; case StackTrace::TAG_DEALLOC: Printf(" %sUninitialized value was created by a heap deallocation%s\n", - d.Origin(), d.End()); + d.Origin(), d.Default()); break; case STACK_TRACE_TAG_POISON: Printf(" %sMemory was marked as uninitialized%s\n", d.Origin(), - d.End()); + d.Default()); break; default: - Printf(" %sUninitialized value was created%s\n", d.Origin(), d.End()); + Printf(" %sUninitialized value was created%s\n", d.Origin(), + d.Default()); break; } stack.Print(); @@ -99,12 +98,12 @@ static void DescribeOrigin(u32 id) { void ReportUMR(StackTrace *stack, u32 origin) { if (!__msan::flags()->report_umrs) return; - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; Decorator d; Printf("%s", d.Warning()); Report("WARNING: MemorySanitizer: use-of-uninitialized-value\n"); - Printf("%s", d.End()); + Printf("%s", d.Default()); stack->Print(); if (origin) { DescribeOrigin(origin); @@ -113,14 +112,14 @@ void ReportUMR(StackTrace *stack, u32 origin) { } void ReportExpectedUMRNotFound(StackTrace *stack) { - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; Printf("WARNING: Expected use of uninitialized value not found\n"); stack->Print(); } void ReportStats() { - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; if (__msan_get_track_origins() > 0) { StackDepotStats *stack_depot_stats = StackDepotGetStats(); @@ -138,13 +137,13 @@ void ReportStats() { } void ReportAtExitStatistics() { - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; if (msan_report_count > 0) { Decorator d; Printf("%s", d.Warning()); Printf("MemorySanitizer: %d warnings reported.\n", msan_report_count); - Printf("%s", d.End()); + Printf("%s", d.Default()); } } @@ -203,7 +202,7 @@ void DescribeMemoryRange(const void *x, uptr size) { Decorator d; Printf("%s", d.Warning()); Printf("Shadow map of [%p, %p), %zu bytes:\n", start, end, end - start); - Printf("%s", d.End()); + Printf("%s", d.Default()); while (s < e) { // Line start. if (pos % 16 == 0) { @@ -265,7 +264,7 @@ void ReportUMRInsideAddressRange(const char *what, const void *start, uptr size, Printf("%s", d.Warning()); Printf("%sUninitialized bytes in %s%s%s at offset %zu inside [%p, %zu)%s\n", d.Warning(), d.Name(), what, d.Warning(), offset, start, size, - d.End()); + d.Default()); if (__sanitizer::Verbosity()) DescribeMemoryRange(start, size); } diff --git a/contrib/compiler-rt/lib/profile/GCDAProfiling.c b/contrib/compiler-rt/lib/profile/GCDAProfiling.c index 138af6ec4033..ef299f002e20 100644 --- a/contrib/compiler-rt/lib/profile/GCDAProfiling.c +++ b/contrib/compiler-rt/lib/profile/GCDAProfiling.c @@ -20,9 +20,6 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfilingPort.h" -#include "InstrProfilingUtil.h" - #include <errno.h> #include <fcntl.h> #include <stdio.h> @@ -34,9 +31,6 @@ #else #include <sys/mman.h> #include <sys/file.h> -#ifndef MAP_FILE -#define MAP_FILE 0 -#endif #endif #if defined(__FreeBSD__) && defined(__i386__) @@ -62,6 +56,9 @@ typedef unsigned int uint32_t; typedef unsigned long long uint64_t; #endif +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + /* #define DEBUG_GCDAPROFILING */ /* @@ -238,17 +235,17 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4], /* Try just opening the file. */ new_file = 0; - fd = open(filename, O_RDWR); + fd = open(filename, O_RDWR | O_BINARY); if (fd == -1) { /* Try opening the file, creating it if necessary. */ new_file = 1; mode = "w+b"; - fd = open(filename, O_RDWR | O_CREAT, 0644); + fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); if (fd == -1) { /* Try creating the directories first then opening the file. */ __llvm_profile_recursive_mkdir(filename); - fd = open(filename, O_RDWR | O_CREAT, 0644); + fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); if (fd == -1) { /* Bah! It's hopeless. */ int errnum = errno; @@ -263,7 +260,7 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4], * same GCDA. This can fail if the filesystem doesn't support it, but in that * case we'll just carry on with the old racy behaviour and hope for the best. */ - flock(fd, LOCK_EX); + lprofLockFd(fd); output_file = fdopen(fd, mode); /* Initialize the write buffer. */ @@ -462,7 +459,7 @@ void llvm_gcda_end_file() { unmap_file(); } - flock(fd, LOCK_UN); + lprofUnlockFd(fd); fclose(output_file); output_file = NULL; write_buffer = NULL; diff --git a/contrib/compiler-rt/lib/profile/InstrProfData.inc b/contrib/compiler-rt/lib/profile/InstrProfData.inc index be0dd4ad04bf..6a98dc7b9b85 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfData.inc +++ b/contrib/compiler-rt/lib/profile/InstrProfData.inc @@ -628,9 +628,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 4 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 4 +#define INSTR_PROF_INDEX_VERSION 5 /* Coverage mapping format vresion (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 1 +#define INSTR_PROF_COVMAP_VERSION 2 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 diff --git a/contrib/compiler-rt/lib/profile/InstrProfiling.c b/contrib/compiler-rt/lib/profile/InstrProfiling.c index fe66fec50658..00b31e1ee7d4 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfiling.c +++ b/contrib/compiler-rt/lib/profile/InstrProfiling.c @@ -7,12 +7,14 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" #include <limits.h> #include <stdio.h> #include <stdlib.h> #include <string.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + #define INSTR_PROF_VALUE_PROF_DATA #include "InstrProfData.inc" diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c index d038bb9cb5b0..d7c0abbc16e3 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c @@ -7,9 +7,6 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" -#include "InstrProfilingUtil.h" #include <errno.h> #include <stdio.h> #include <stdlib.h> @@ -31,6 +28,10 @@ #endif #endif +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + /* From where is profile name specified. * The order the enumerators define their * precedence. Re-order them may lead to @@ -85,7 +86,6 @@ typedef struct lprofFilename { COMPILER_RT_WEAK lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0}, 0, 0, 0, PNS_unknown}; -int getpid(void); static int getCurFilenameLength(); static const char *getCurFilename(char *FilenameBuf); static unsigned doMerging() { return lprofCurFilename.MergePoolSize; } @@ -325,7 +325,7 @@ static int parseFilenamePattern(const char *FilenamePat, if (FilenamePat[I] == '%') { if (FilenamePat[++I] == 'p') { if (!NumPids++) { - if (snprintf(PidChars, MAX_PID_SIZE, "%d", getpid()) <= 0) { + if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) { PROF_WARN("Unable to get pid for filename pattern %s. Using the " "default name.", FilenamePat); @@ -519,8 +519,10 @@ void __llvm_profile_initialize_file(void) { EnvFilenamePat = getFilenamePatFromEnv(); if (EnvFilenamePat) { - SelectedPat = EnvFilenamePat; - PNS = PNS_environment; + /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid + at the moment when __llvm_profile_write_file() gets executed. */ + parseAndSetFilename(EnvFilenamePat, PNS_environment, 1); + return; } else if (hasCommandLineOverrider) { SelectedPat = INSTR_PROF_PROFILE_NAME_VAR; PNS = PNS_command_line; diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h b/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h index 36490ef7d433..40540ab5f205 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -10,8 +10,9 @@ #ifndef PROFILE_INSTRPROFILING_INTERNALH_ #define PROFILE_INSTRPROFILING_INTERNALH_ +#include <stddef.h> + #include "InstrProfiling.h" -#include "stddef.h" /*! * \brief Write instrumentation data to the given buffer, given explicit diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index b6c780ff514f..89f1ab4cfeda 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -7,11 +7,13 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" +#if defined(__linux__) || defined(__FreeBSD__) || \ + (defined(__sun__) && defined(__svr4__)) -#if defined(__linux__) || defined(__FreeBSD__) #include <stdlib.h> +#include "InstrProfiling.h" + #define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) #define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME) #define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME) diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index b25966487e91..a339abc7f883 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -7,12 +7,13 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" - -#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ + !(defined(__sun__) && defined(__svr4__)) #include <stdlib.h> +#include "InstrProfiling.h" + static const __llvm_profile_data *DataFirst = NULL; static const __llvm_profile_data *DataLast = NULL; static const char *NamesFirst = NULL; diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPort.h b/contrib/compiler-rt/lib/profile/InstrProfilingPort.h index 5789351956b9..cd1264b0e575 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPort.h +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPort.h @@ -7,6 +7,9 @@ |* \*===----------------------------------------------------------------------===*/ +/* This header must be included after all others so it can provide fallback + definitions for stuff missing in system headers. */ + #ifndef PROFILE_INSTRPROFILING_PORT_H_ #define PROFILE_INSTRPROFILING_PORT_H_ @@ -44,9 +47,6 @@ #define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1)) #else #define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len) -#ifndef _MSC_VER -#define COMPILER_RT_HAS_UNAME 1 -#endif #endif #if COMPILER_RT_HAS_ATOMICS == 1 @@ -107,6 +107,14 @@ #define PROF_NOTE(Format, ...) \ fprintf(stderr, "LLVM Profile Note: " Format, __VA_ARGS__); +#ifndef MAP_FILE +#define MAP_FILE 0 +#endif + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + #if defined(__FreeBSD__) #include <inttypes.h> diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c index fb68f30a5e1f..110562383412 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c @@ -7,13 +7,10 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfilingUtil.h" -#include "InstrProfiling.h" - #ifdef _WIN32 #include <direct.h> -#include <io.h> #include <windows.h> +#include "WindowsMMap.h" #else #include <sys/stat.h> #include <sys/types.h> @@ -34,6 +31,9 @@ #include <sys/prctl.h> #endif +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + COMPILER_RT_VISIBILITY void __llvm_profile_recursive_mkdir(char *path) { int i; @@ -86,16 +86,16 @@ COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { #elif defined(COMPILER_RT_HAS_UNAME) COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { struct utsname N; - int R; - if (!(R = uname(&N))) + int R = uname(&N); + if (R >= 0) { strncpy(Name, N.nodename, Len); + return 0; + } return R; } #endif -COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { - FILE *f; - int fd; +COMPILER_RT_VISIBILITY int lprofLockFd(int fd) { #ifdef COMPILER_RT_HAS_FCNTL_LCK struct flock s_flock; @@ -103,21 +103,59 @@ COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { s_flock.l_start = 0; s_flock.l_len = 0; /* Until EOF. */ s_flock.l_pid = getpid(); - s_flock.l_type = F_WRLCK; - fd = open(ProfileName, O_RDWR | O_CREAT, 0666); - if (fd < 0) - return NULL; while (fcntl(fd, F_SETLKW, &s_flock) == -1) { if (errno != EINTR) { if (errno == ENOLCK) { - PROF_WARN("Data may be corrupted during profile merging : %s\n", - "Fail to obtain file lock due to system limit."); + return -1; } break; } } + return 0; +#else + flock(fd, LOCK_EX); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) { +#ifdef COMPILER_RT_HAS_FCNTL_LCK + struct flock s_flock; + + s_flock.l_whence = SEEK_SET; + s_flock.l_start = 0; + s_flock.l_len = 0; /* Until EOF. */ + s_flock.l_pid = getpid(); + s_flock.l_type = F_UNLCK; + + while (fcntl(fd, F_SETLKW, &s_flock) == -1) { + if (errno != EINTR) { + if (errno == ENOLCK) { + return -1; + } + break; + } + } + return 0; +#else + flock(fd, LOCK_UN); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { + FILE *f; + int fd; +#ifdef COMPILER_RT_HAS_FCNTL_LCK + fd = open(ProfileName, O_RDWR | O_CREAT, 0666); + if (fd < 0) + return NULL; + + if (lprofLockFd(fd) != 0) + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); f = fdopen(fd, "r+b"); #elif defined(_WIN32) diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h index 9698599606e0..2f2ea1b876a8 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h +++ b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h @@ -16,6 +16,9 @@ /*! \brief Create a directory tree. */ void __llvm_profile_recursive_mkdir(char *Pathname); +int lprofLockFd(int fd); +int lprofUnlockFd(int fd); + /*! Open file \c Filename for read+write with write * lock for exclusive access. The caller will block * if the lock is already held by another process. */ diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c index 44263da80097..3db0de8a6723 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c @@ -7,13 +7,15 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" -#include "InstrProfilingUtil.h" /* For PS4 getenv shim. */ #include <limits.h> #include <stdio.h> #include <stdlib.h> #include <string.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + #define INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_COMMON_API_IMPL #include "InstrProfData.inc" @@ -22,7 +24,7 @@ static int hasStaticCounters = 1; static int OutOfNodesWarnings = 0; static int hasNonDefaultValsPerSite = 0; #define INSTR_PROF_MAX_VP_WARNS 10 -#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 8 +#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16 #define INSTR_PROF_VNODE_POOL_SIZE 1024 #ifndef _MSC_VER diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c b/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c index d4c9b9bd663c..7c6061d2c65a 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -7,14 +7,15 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" #ifdef _MSC_VER /* For _alloca */ #include <malloc.h> #endif #include <string.h> +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + #define INSTR_PROF_VALUE_PROF_DATA #include "InstrProfData.inc" diff --git a/contrib/compiler-rt/lib/profile/WindowsMMap.c b/contrib/compiler-rt/lib/profile/WindowsMMap.c index f81d7da5389e..dc87a888ae7b 100644 --- a/contrib/compiler-rt/lib/profile/WindowsMMap.c +++ b/contrib/compiler-rt/lib/profile/WindowsMMap.c @@ -18,11 +18,12 @@ #if defined(_WIN32) #include "WindowsMMap.h" -#include "InstrProfiling.h" #define WIN32_LEAN_AND_MEAN #include <windows.h> +#include "InstrProfiling.h" + #ifdef __USE_FILE_OFFSET64 # define DWORD_HI(x) (x >> 32) # define DWORD_LO(x) ((x) & 0xffffffff) @@ -120,9 +121,61 @@ int msync(void *addr, size_t length, int flags) } COMPILER_RT_VISIBILITY -int flock(int fd, int operation) -{ - return -1; /* Not supported. */ +int lock(HANDLE handle, DWORD lockType, BOOL blocking) { + DWORD flags = lockType; + if (!blocking) + flags |= LOCKFILE_FAIL_IMMEDIATELY; + + OVERLAPPED overlapped; + ZeroMemory(&overlapped, sizeof(OVERLAPPED)); + overlapped.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + BOOL result = LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped); + if (!result) { + DWORD dw = GetLastError(); + + // In non-blocking mode, return an error if the file is locked. + if (!blocking && dw == ERROR_LOCK_VIOLATION) + return -1; // EWOULDBLOCK + + // If the error is ERROR_IO_PENDING, we need to wait until the operation + // finishes. Otherwise, we return an error. + if (dw != ERROR_IO_PENDING) + return -1; + + DWORD dwNumBytes; + if (!GetOverlappedResult(handle, &overlapped, &dwNumBytes, TRUE)) + return -1; + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int flock(int fd, int operation) { + HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) + return -1; + + BOOL blocking = (operation & LOCK_NB) == 0; + int op = operation & ~LOCK_NB; + + switch (op) { + case LOCK_EX: + return lock(handle, LOCKFILE_EXCLUSIVE_LOCK, blocking); + + case LOCK_SH: + return lock(handle, 0, blocking); + + case LOCK_UN: + if (!UnlockFile(handle, 0, 0, MAXDWORD, MAXDWORD)) + return -1; + break; + + default: + return -1; + } + + return 0; } #undef DWORD_HI diff --git a/contrib/compiler-rt/lib/safestack/safestack.cc b/contrib/compiler-rt/lib/safestack/safestack.cc index b194b6cfa8f9..d783cd5a9b29 100644 --- a/contrib/compiler-rt/lib/safestack/safestack.cc +++ b/contrib/compiler-rt/lib/safestack/safestack.cc @@ -21,7 +21,9 @@ #include <unistd.h> #include <sys/resource.h> #include <sys/types.h> +#if !defined(__NetBSD__) #include <sys/user.h> +#endif #include "interception/interception.h" #include "sanitizer_common/sanitizer_common.h" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc index 84f523c5e431..fc4f7a75ae34 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc @@ -178,11 +178,13 @@ void InternalFree(void *addr, InternalAllocatorCache *cache) { } // LowLevelAllocator +constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; +static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; static LowLevelAllocateCallback low_level_alloc_callback; void *LowLevelAllocator::Allocate(uptr size) { // Align allocation size. - size = RoundUpTo(size, 8); + size = RoundUpTo(size, low_level_alloc_min_alignment); if (allocated_end_ - allocated_current_ < (sptr)size) { uptr size_to_allocate = Max(size, GetPageSizeCached()); allocated_current_ = @@ -199,6 +201,11 @@ void *LowLevelAllocator::Allocate(uptr size) { return res; } +void SetLowLevelAllocateMinAlignment(uptr alignment) { + CHECK(IsPowerOfTwo(alignment)); + low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment); +} + void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) { low_level_alloc_callback = callback; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h index 8c5696ea789c..38368361de6f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -56,6 +56,19 @@ struct NoOpMapUnmapCallback { // Callback type for iterating over chunks. typedef void (*ForEachChunkCallback)(uptr chunk, void *arg); +INLINE u32 Rand(u32 *state) { // ANSI C linear congruential PRNG. + return (*state = *state * 1103515245 + 12345) >> 16; +} + +INLINE u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n) + +template<typename T> +INLINE void RandomShuffle(T *a, u32 n, u32 *rand_state) { + if (n <= 1) return; + for (u32 i = n - 1; i > 0; i--) + Swap(a[i], a[RandN(rand_state, i + 1)]); +} + #include "sanitizer_allocator_size_class_map.h" #include "sanitizer_allocator_stats.h" #include "sanitizer_allocator_primary64.h" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc new file mode 100644 index 000000000000..dc263dbef5fc --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc @@ -0,0 +1,23 @@ +//===-- sanitizer_allocator_checks.cc ---------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory +// allocators. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_errno.h" + +namespace __sanitizer { + +void SetErrnoToENOMEM() { + errno = errno_ENOMEM; +} + +} // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h index 202916eae348..b61a8b2eb3b6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h @@ -15,17 +15,22 @@ #ifndef SANITIZER_ALLOCATOR_CHECKS_H #define SANITIZER_ALLOCATOR_CHECKS_H -#include "sanitizer_errno.h" #include "sanitizer_internal_defs.h" #include "sanitizer_common.h" #include "sanitizer_platform.h" namespace __sanitizer { +// The following is defined in a separate compilation unit to avoid pulling in +// sanitizer_errno.h in this header, which leads to conflicts when other system +// headers include errno.h. This is usually the result of an unlikely event, +// and as such we do not care as much about having it inlined. +void SetErrnoToENOMEM(); + // A common errno setting logic shared by almost all sanitizer allocator APIs. INLINE void *SetErrnoOnNull(void *ptr) { if (UNLIKELY(!ptr)) - errno = errno_ENOMEM; + SetErrnoToENOMEM(); return ptr; } @@ -59,6 +64,12 @@ INLINE bool CheckForCallocOverflow(uptr size, uptr n) { return (max / size) < n; } +// Returns true if the size passed to pvalloc overflows when rounded to the next +// multiple of page_size. +INLINE bool CheckForPvallocOverflow(uptr size, uptr page_size) { + return RoundUpTo(size, page_size) < size; +} + } // namespace __sanitizer #endif // SANITIZER_ALLOCATOR_CHECKS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h index efd25cadfe74..0d8a2a174bb5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h @@ -77,6 +77,10 @@ class CombinedAllocator { primary_.SetReleaseToOSIntervalMs(release_to_os_interval_ms); } + void ForceReleaseToOS() { + primary_.ForceReleaseToOS(); + } + void Deallocate(AllocatorCache *cache, void *p) { if (!p) return; if (primary_.PointerIsMine(p)) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h index 13910e719e78..2f5ce3151ee4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h @@ -39,6 +39,9 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_free_hook(void *ptr); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_purge_allocator(); + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_print_memory_profile(uptr top_percent, uptr max_number_of_contexts); } // extern "C" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h index ec0742c20be2..1b3c2c0c19d1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h @@ -26,9 +26,6 @@ struct SizeClassAllocatorLocalCache template <class SizeClassAllocator> struct SizeClassAllocator64LocalCache { typedef SizeClassAllocator Allocator; - static const uptr kNumClasses = SizeClassAllocator::kNumClasses; - typedef typename Allocator::SizeClassMapT SizeClassMap; - typedef typename Allocator::CompactPtrT CompactPtrT; void Init(AllocatorGlobalStats *s) { stats_.Init(); @@ -76,14 +73,18 @@ struct SizeClassAllocator64LocalCache { } void Drain(SizeClassAllocator *allocator) { - for (uptr class_id = 0; class_id < kNumClasses; class_id++) { - PerClass *c = &per_class_[class_id]; + for (uptr i = 0; i < kNumClasses; i++) { + PerClass *c = &per_class_[i]; while (c->count > 0) - Drain(c, allocator, class_id, c->count); + Drain(c, allocator, i, c->count); } } - // private: + private: + typedef typename Allocator::SizeClassMapT SizeClassMap; + static const uptr kNumClasses = SizeClassMap::kNumClasses; + typedef typename Allocator::CompactPtrT CompactPtrT; + struct PerClass { u32 count; u32 max_count; @@ -94,7 +95,7 @@ struct SizeClassAllocator64LocalCache { AllocatorStats stats_; void InitCache() { - if (per_class_[1].max_count) + if (LIKELY(per_class_[1].max_count)) return; for (uptr i = 0; i < kNumClasses; i++) { PerClass *c = &per_class_[i]; @@ -130,7 +131,6 @@ template <class SizeClassAllocator> struct SizeClassAllocator32LocalCache { typedef SizeClassAllocator Allocator; typedef typename Allocator::TransferBatch TransferBatch; - static const uptr kNumClasses = SizeClassAllocator::kNumClasses; void Init(AllocatorGlobalStats *s) { stats_.Init(); @@ -138,6 +138,21 @@ struct SizeClassAllocator32LocalCache { s->Register(&stats_); } + // Returns a TransferBatch suitable for class_id. + TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator, + TransferBatch *b) { + if (uptr batch_class_id = per_class_[class_id].batch_class_id) + return (TransferBatch*)Allocate(allocator, batch_class_id); + return b; + } + + // Destroys TransferBatch b. + void DestroyBatch(uptr class_id, SizeClassAllocator *allocator, + TransferBatch *b) { + if (uptr batch_class_id = per_class_[class_id].batch_class_id) + Deallocate(allocator, batch_class_id, b); + } + void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) { Drain(allocator); if (s) @@ -173,66 +188,57 @@ struct SizeClassAllocator32LocalCache { } void Drain(SizeClassAllocator *allocator) { - for (uptr class_id = 0; class_id < kNumClasses; class_id++) { - PerClass *c = &per_class_[class_id]; + for (uptr i = 0; i < kNumClasses; i++) { + PerClass *c = &per_class_[i]; while (c->count > 0) - Drain(allocator, class_id); + Drain(allocator, i); } } - // private: - typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap; + private: + typedef typename Allocator::SizeClassMapT SizeClassMap; + static const uptr kBatchClassID = SizeClassMap::kBatchClassID; + static const uptr kNumClasses = SizeClassMap::kNumClasses; + // If kUseSeparateSizeClassForBatch is true, all TransferBatch objects are + // allocated from kBatchClassID size class (except for those that are needed + // for kBatchClassID itself). The goal is to have TransferBatches in a totally + // different region of RAM to improve security. + static const bool kUseSeparateSizeClassForBatch = + Allocator::kUseSeparateSizeClassForBatch; + struct PerClass { uptr count; uptr max_count; uptr class_size; - uptr class_id_for_transfer_batch; + uptr batch_class_id; void *batch[2 * TransferBatch::kMaxNumCached]; }; PerClass per_class_[kNumClasses]; AllocatorStats stats_; void InitCache() { - if (per_class_[1].max_count) + if (LIKELY(per_class_[1].max_count)) return; - // TransferBatch class is declared in SizeClassAllocator. - uptr class_id_for_transfer_batch = - SizeClassMap::ClassID(sizeof(TransferBatch)); + const uptr batch_class_id = SizeClassMap::ClassID(sizeof(TransferBatch)); for (uptr i = 0; i < kNumClasses; i++) { PerClass *c = &per_class_[i]; uptr max_cached = TransferBatch::MaxCached(i); c->max_count = 2 * max_cached; c->class_size = Allocator::ClassIdToSize(i); - // We transfer chunks between central and thread-local free lists in - // batches. For small size classes we allocate batches separately. For - // large size classes we may use one of the chunks to store the batch. - // sizeof(TransferBatch) must be a power of 2 for more efficient - // allocation. - c->class_id_for_transfer_batch = (c->class_size < + // Precompute the class id to use to store batches for the current class + // id. 0 means the class size is large enough to store a batch within one + // of the chunks. If using a separate size class, it will always be + // kBatchClassID, except for kBatchClassID itself. + if (kUseSeparateSizeClassForBatch) { + c->batch_class_id = (i == kBatchClassID) ? 0 : kBatchClassID; + } else { + c->batch_class_id = (c->class_size < TransferBatch::AllocationSizeRequiredForNElements(max_cached)) ? - class_id_for_transfer_batch : 0; + batch_class_id : 0; + } } } - // Returns a TransferBatch suitable for class_id. - // For small size classes allocates the batch from the allocator. - // For large size classes simply returns b. - TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator, - TransferBatch *b) { - if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch) - return (TransferBatch*)Allocate(allocator, batch_class_id); - return b; - } - - // Destroys TransferBatch b. - // For small size classes deallocates b to the allocator. - // Does notthing for large size classes. - void DestroyBatch(uptr class_id, SizeClassAllocator *allocator, - TransferBatch *b) { - if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch) - Deallocate(allocator, batch_class_id, b); - } - NOINLINE bool Refill(SizeClassAllocator *allocator, uptr class_id) { InitCache(); PerClass *c = &per_class_[class_id]; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h index e85821543d27..6c682aff64a2 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -41,6 +41,7 @@ template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache; struct SizeClassAllocator32FlagMasks { // Bit masks. enum { kRandomShuffleChunks = 1, + kUseSeparateSizeClassForBatch = 2, }; }; @@ -55,8 +56,10 @@ class SizeClassAllocator32 { typedef typename Params::ByteMap ByteMap; typedef typename Params::MapUnmapCallback MapUnmapCallback; - static const bool kRandomShuffleChunks = - Params::kFlags & SizeClassAllocator32FlagMasks::kRandomShuffleChunks; + static const bool kRandomShuffleChunks = Params::kFlags & + SizeClassAllocator32FlagMasks::kRandomShuffleChunks; + static const bool kUseSeparateSizeClassForBatch = Params::kFlags & + SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch; struct TransferBatch { static const uptr kMaxNumCached = SizeClassMap::kMaxNumCachedHint - 2; @@ -94,11 +97,11 @@ class SizeClassAllocator32 { static const uptr kBatchSize = sizeof(TransferBatch); COMPILER_CHECK((kBatchSize & (kBatchSize - 1)) == 0); - COMPILER_CHECK(sizeof(TransferBatch) == - SizeClassMap::kMaxNumCachedHint * sizeof(uptr)); + COMPILER_CHECK(kBatchSize == SizeClassMap::kMaxNumCachedHint * sizeof(uptr)); static uptr ClassIdToSize(uptr class_id) { - return SizeClassMap::Size(class_id); + return (class_id == SizeClassMap::kBatchClassID) ? + kBatchSize : SizeClassMap::Size(class_id); } typedef SizeClassAllocator32<Params> ThisT; @@ -117,6 +120,10 @@ class SizeClassAllocator32 { // This is empty here. Currently only implemented in 64-bit allocator. } + void ForceReleaseToOS() { + // Currently implemented in 64-bit allocator only. + } + void *MapWithCallback(uptr size) { void *res = MmapOrDie(size, "SizeClassAllocator32"); MapUnmapCallback().OnMap((uptr)res, size); @@ -161,9 +168,9 @@ class SizeClassAllocator32 { NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id, TransferBatch *b) { CHECK_LT(class_id, kNumClasses); + CHECK_GT(b->Count(), 0); SizeClassInfo *sci = GetSizeClassInfo(class_id); SpinMutexLock l(&sci->mutex); - CHECK_GT(b->Count(), 0); sci->free_list.push_front(b); } @@ -261,7 +268,8 @@ class SizeClassAllocator32 { struct SizeClassInfo { SpinMutex mutex; IntrusiveList<TransferBatch> free_list; - char padding[kCacheLineSize - sizeof(uptr) - + u32 rand_state; + char padding[kCacheLineSize - 2 * sizeof(uptr) - sizeof(IntrusiveList<TransferBatch>)]; }; COMPILER_CHECK(sizeof(SizeClassInfo) == kCacheLineSize); @@ -294,29 +302,62 @@ class SizeClassAllocator32 { return &size_class_info_array[class_id]; } + bool PopulateBatches(AllocatorCache *c, SizeClassInfo *sci, uptr class_id, + TransferBatch **current_batch, uptr max_count, + uptr *pointers_array, uptr count) { + // If using a separate class for batches, we do not need to shuffle it. + if (kRandomShuffleChunks && (!kUseSeparateSizeClassForBatch || + class_id != SizeClassMap::kBatchClassID)) + RandomShuffle(pointers_array, count, &sci->rand_state); + TransferBatch *b = *current_batch; + for (uptr i = 0; i < count; i++) { + if (!b) { + b = c->CreateBatch(class_id, this, (TransferBatch*)pointers_array[i]); + if (UNLIKELY(!b)) + return false; + b->Clear(); + } + b->Add((void*)pointers_array[i]); + if (b->Count() == max_count) { + sci->free_list.push_back(b); + b = nullptr; + } + } + *current_batch = b; + return true; + } + bool PopulateFreeList(AllocatorStats *stat, AllocatorCache *c, SizeClassInfo *sci, uptr class_id) { uptr size = ClassIdToSize(class_id); uptr reg = AllocateRegion(stat, class_id); if (UNLIKELY(!reg)) return false; + if (kRandomShuffleChunks) + if (UNLIKELY(sci->rand_state == 0)) + // The random state is initialized from ASLR (PIE) and time. + sci->rand_state = reinterpret_cast<uptr>(sci) ^ NanoTime(); uptr n_chunks = kRegionSize / (size + kMetadataSize); uptr max_count = TransferBatch::MaxCached(class_id); CHECK_GT(max_count, 0); TransferBatch *b = nullptr; + const uptr kShuffleArraySize = 48; + uptr shuffle_array[kShuffleArraySize]; + uptr count = 0; for (uptr i = reg; i < reg + n_chunks * size; i += size) { - if (!b) { - b = c->CreateBatch(class_id, this, (TransferBatch*)i); - if (UNLIKELY(!b)) + shuffle_array[count++] = i; + if (count == kShuffleArraySize) { + if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count, + shuffle_array, count))) return false; - b->Clear(); - } - b->Add((void*)i); - if (b->Count() == max_count) { - sci->free_list.push_back(b); - b = nullptr; + count = 0; } } + if (count) { + if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count, + shuffle_array, count))) + return false; + } if (b) { CHECK_GT(b->Count(), 0); sci->free_list.push_back(b); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h index 0c2e72ce7eb1..651a64b04f2e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -62,21 +62,20 @@ class SizeClassAllocator64 { // as a 4-byte integer (offset from the region start shifted right by 4). typedef u32 CompactPtrT; static const uptr kCompactPtrScale = 4; - CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) { + CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) const { return static_cast<CompactPtrT>((ptr - base) >> kCompactPtrScale); } - uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) { + uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) const { return base + (static_cast<uptr>(ptr32) << kCompactPtrScale); } void Init(s32 release_to_os_interval_ms) { uptr TotalSpaceSize = kSpaceSize + AdditionalSize(); if (kUsingConstantSpaceBeg) { - CHECK_EQ(kSpaceBeg, reinterpret_cast<uptr>( - MmapFixedNoAccess(kSpaceBeg, TotalSpaceSize))); + CHECK_EQ(kSpaceBeg, address_range.Init(TotalSpaceSize, AllocatorName(), + kSpaceBeg)); } else { - NonConstSpaceBeg = - reinterpret_cast<uptr>(MmapNoAccess(TotalSpaceSize)); + NonConstSpaceBeg = address_range.Init(TotalSpaceSize, AllocatorName()); CHECK_NE(NonConstSpaceBeg, ~(uptr)0); } SetReleaseToOSIntervalMs(release_to_os_interval_ms); @@ -92,6 +91,13 @@ class SizeClassAllocator64 { memory_order_relaxed); } + void ForceReleaseToOS() { + for (uptr class_id = 1; class_id < kNumClasses; class_id++) { + BlockingMutexLock l(&GetRegionInfo(class_id)->mutex); + MaybeReleaseToOS(class_id, true /*force*/); + } + } + static bool CanAllocate(uptr size, uptr alignment) { return size <= SizeClassMap::kMaxSize && alignment <= SizeClassMap::kMaxSize; @@ -116,7 +122,7 @@ class SizeClassAllocator64 { region->num_freed_chunks = new_num_freed_chunks; region->stats.n_freed += n_chunks; - MaybeReleaseToOS(class_id); + MaybeReleaseToOS(class_id, false /*force*/); } NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id, @@ -155,7 +161,7 @@ class SizeClassAllocator64 { space_beg; } - uptr GetRegionBeginBySizeClass(uptr class_id) { + uptr GetRegionBeginBySizeClass(uptr class_id) const { return SpaceBeg() + kRegionSize * class_id; } @@ -223,30 +229,39 @@ class SizeClassAllocator64 { uptr avail_chunks = region->allocated_user / ClassIdToSize(class_id); Printf( "%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd " - "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd\n", + "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd " + "last released: %6zdK region: 0x%zx\n", region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id), region->mapped_user >> 10, region->stats.n_allocated, region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks, - rss >> 10, region->rtoi.num_releases); + rss >> 10, region->rtoi.num_releases, + region->rtoi.last_released_bytes >> 10, + SpaceBeg() + kRegionSize * class_id); } void PrintStats() { + uptr rss_stats[kNumClasses]; + for (uptr class_id = 0; class_id < kNumClasses; class_id++) + rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id; + GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses); + uptr total_mapped = 0; + uptr total_rss = 0; uptr n_allocated = 0; uptr n_freed = 0; for (uptr class_id = 1; class_id < kNumClasses; class_id++) { RegionInfo *region = GetRegionInfo(class_id); - total_mapped += region->mapped_user; + if (region->mapped_user != 0) { + total_mapped += region->mapped_user; + total_rss += rss_stats[class_id]; + } n_allocated += region->stats.n_allocated; n_freed += region->stats.n_freed; } - Printf("Stats: SizeClassAllocator64: %zdM mapped in %zd allocations; " - "remains %zd\n", - total_mapped >> 20, n_allocated, n_allocated - n_freed); - uptr rss_stats[kNumClasses]; - for (uptr class_id = 0; class_id < kNumClasses; class_id++) - rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id; - GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses); + + Printf("Stats: SizeClassAllocator64: %zdM mapped (%zdM rss) in " + "%zd allocations; remains %zd\n", total_mapped >> 20, + total_rss >> 20, n_allocated, n_allocated - n_freed); for (uptr class_id = 1; class_id < kNumClasses; class_id++) PrintStats(class_id, rss_stats[class_id]); } @@ -294,7 +309,243 @@ class SizeClassAllocator64 { static const uptr kNumClasses = SizeClassMap::kNumClasses; static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded; + // A packed array of counters. Each counter occupies 2^n bits, enough to store + // counter's max_value. Ctor will try to allocate the required buffer via + // mapper->MapPackedCounterArrayBuffer and the caller is expected to check + // whether the initialization was successful by checking IsAllocated() result. + // For the performance sake, none of the accessors check the validity of the + // arguments, it is assumed that index is always in [0, n) range and the value + // is not incremented past max_value. + template<class MemoryMapperT> + class PackedCounterArray { + public: + PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper) + : n(num_counters), memory_mapper(mapper) { + CHECK_GT(num_counters, 0); + CHECK_GT(max_value, 0); + constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL; + // Rounding counter storage size up to the power of two allows for using + // bit shifts calculating particular counter's index and offset. + uptr counter_size_bits = + RoundUpToPowerOfTwo(MostSignificantSetBitIndex(max_value) + 1); + CHECK_LE(counter_size_bits, kMaxCounterBits); + counter_size_bits_log = Log2(counter_size_bits); + counter_mask = ~0ULL >> (kMaxCounterBits - counter_size_bits); + + uptr packing_ratio = kMaxCounterBits >> counter_size_bits_log; + CHECK_GT(packing_ratio, 0); + packing_ratio_log = Log2(packing_ratio); + bit_offset_mask = packing_ratio - 1; + + buffer_size = + (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) * + sizeof(*buffer); + buffer = reinterpret_cast<u64*>( + memory_mapper->MapPackedCounterArrayBuffer(buffer_size)); + } + ~PackedCounterArray() { + if (buffer) { + memory_mapper->UnmapPackedCounterArrayBuffer( + reinterpret_cast<uptr>(buffer), buffer_size); + } + } + + bool IsAllocated() const { + return !!buffer; + } + + u64 GetCount() const { + return n; + } + + uptr Get(uptr i) const { + DCHECK_LT(i, n); + uptr index = i >> packing_ratio_log; + uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log; + return (buffer[index] >> bit_offset) & counter_mask; + } + + void Inc(uptr i) const { + DCHECK_LT(Get(i), counter_mask); + uptr index = i >> packing_ratio_log; + uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log; + buffer[index] += 1ULL << bit_offset; + } + + void IncRange(uptr from, uptr to) const { + DCHECK_LE(from, to); + for (uptr i = from; i <= to; i++) + Inc(i); + } + + private: + const u64 n; + u64 counter_size_bits_log; + u64 counter_mask; + u64 packing_ratio_log; + u64 bit_offset_mask; + + MemoryMapperT* const memory_mapper; + u64 buffer_size; + u64* buffer; + }; + + template<class MemoryMapperT> + class FreePagesRangeTracker { + public: + explicit FreePagesRangeTracker(MemoryMapperT* mapper) + : memory_mapper(mapper), + page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)), + in_the_range(false), current_page(0), current_range_start_page(0) {} + + void NextPage(bool freed) { + if (freed) { + if (!in_the_range) { + current_range_start_page = current_page; + in_the_range = true; + } + } else { + CloseOpenedRange(); + } + current_page++; + } + + void Done() { + CloseOpenedRange(); + } + + private: + void CloseOpenedRange() { + if (in_the_range) { + memory_mapper->ReleasePageRangeToOS( + current_range_start_page << page_size_scaled_log, + current_page << page_size_scaled_log); + in_the_range = false; + } + } + + MemoryMapperT* const memory_mapper; + const uptr page_size_scaled_log; + bool in_the_range; + uptr current_page; + uptr current_range_start_page; + }; + + // Iterates over the free_array to identify memory pages containing freed + // chunks only and returns these pages back to OS. + // allocated_pages_count is the total number of pages allocated for the + // current bucket. + template<class MemoryMapperT> + static void ReleaseFreeMemoryToOS(CompactPtrT *free_array, + uptr free_array_count, uptr chunk_size, + uptr allocated_pages_count, + MemoryMapperT *memory_mapper) { + const uptr page_size = GetPageSizeCached(); + + // Figure out the number of chunks per page and whether we can take a fast + // path (the number of chunks per page is the same for all pages). + uptr full_pages_chunk_count_max; + bool same_chunk_count_per_page; + if (chunk_size <= page_size && page_size % chunk_size == 0) { + // Same number of chunks per page, no cross overs. + full_pages_chunk_count_max = page_size / chunk_size; + same_chunk_count_per_page = true; + } else if (chunk_size <= page_size && page_size % chunk_size != 0 && + chunk_size % (page_size % chunk_size) == 0) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks, but all pages contain the same + // number of chunks. + full_pages_chunk_count_max = page_size / chunk_size + 1; + same_chunk_count_per_page = true; + } else if (chunk_size <= page_size) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks. + full_pages_chunk_count_max = page_size / chunk_size + 2; + same_chunk_count_per_page = false; + } else if (chunk_size > page_size && chunk_size % page_size == 0) { + // One chunk covers multiple pages, no cross overs. + full_pages_chunk_count_max = 1; + same_chunk_count_per_page = true; + } else if (chunk_size > page_size) { + // One chunk covers multiple pages, Some chunks are crossing page + // boundaries. Some pages contain one chunk, some contain two. + full_pages_chunk_count_max = 2; + same_chunk_count_per_page = false; + } else { + UNREACHABLE("All chunk_size/page_size ratios must be handled."); + } + + PackedCounterArray<MemoryMapperT> counters(allocated_pages_count, + full_pages_chunk_count_max, + memory_mapper); + if (!counters.IsAllocated()) + return; + + const uptr chunk_size_scaled = chunk_size >> kCompactPtrScale; + const uptr page_size_scaled = page_size >> kCompactPtrScale; + const uptr page_size_scaled_log = Log2(page_size_scaled); + + // Iterate over free chunks and count how many free chunks affect each + // allocated page. + if (chunk_size <= page_size && page_size % chunk_size == 0) { + // Each chunk affects one page only. + for (uptr i = 0; i < free_array_count; i++) + counters.Inc(free_array[i] >> page_size_scaled_log); + } else { + // In all other cases chunks might affect more than one page. + for (uptr i = 0; i < free_array_count; i++) { + counters.IncRange( + free_array[i] >> page_size_scaled_log, + (free_array[i] + chunk_size_scaled - 1) >> page_size_scaled_log); + } + } + + // Iterate over pages detecting ranges of pages with chunk counters equal + // to the expected number of chunks for the particular page. + FreePagesRangeTracker<MemoryMapperT> range_tracker(memory_mapper); + if (same_chunk_count_per_page) { + // Fast path, every page has the same number of chunks affecting it. + for (uptr i = 0; i < counters.GetCount(); i++) + range_tracker.NextPage(counters.Get(i) == full_pages_chunk_count_max); + } else { + // Show path, go through the pages keeping count how many chunks affect + // each page. + const uptr pn = + chunk_size < page_size ? page_size_scaled / chunk_size_scaled : 1; + const uptr pnc = pn * chunk_size_scaled; + // The idea is to increment the current page pointer by the first chunk + // size, middle portion size (the portion of the page covered by chunks + // except the first and the last one) and then the last chunk size, adding + // up the number of chunks on the current page and checking on every step + // whether the page boundary was crossed. + uptr prev_page_boundary = 0; + uptr current_boundary = 0; + for (uptr i = 0; i < counters.GetCount(); i++) { + uptr page_boundary = prev_page_boundary + page_size_scaled; + uptr chunks_per_page = pn; + if (current_boundary < page_boundary) { + if (current_boundary > prev_page_boundary) + chunks_per_page++; + current_boundary += pnc; + if (current_boundary < page_boundary) { + chunks_per_page++; + current_boundary += chunk_size_scaled; + } + } + prev_page_boundary = page_boundary; + + range_tracker.NextPage(counters.Get(i) == chunks_per_page); + } + } + range_tracker.Done(); + } + private: + friend class MemoryMapper; + + ReservedAddressRange address_range; + static const char *AllocatorName() { return "sanitizer_allocator"; } + static const uptr kRegionSize = kSpaceSize / kNumClassesRounded; // FreeArray is the array of free-d chunks (stored as 4-byte offsets). // In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize @@ -330,6 +581,7 @@ class SizeClassAllocator64 { uptr n_freed_at_last_release; uptr num_releases; u64 last_release_at_ns; + u64 last_released_bytes; }; struct RegionInfo { @@ -347,30 +599,18 @@ class SizeClassAllocator64 { }; COMPILER_CHECK(sizeof(RegionInfo) >= kCacheLineSize); - u32 Rand(u32 *state) { // ANSI C linear congruential PRNG. - return (*state = *state * 1103515245 + 12345) >> 16; - } - - u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n) - - void RandomShuffle(u32 *a, u32 n, u32 *rand_state) { - if (n <= 1) return; - for (u32 i = n - 1; i > 0; i--) - Swap(a[i], a[RandN(rand_state, i + 1)]); - } - - RegionInfo *GetRegionInfo(uptr class_id) { + RegionInfo *GetRegionInfo(uptr class_id) const { CHECK_LT(class_id, kNumClasses); RegionInfo *regions = reinterpret_cast<RegionInfo *>(SpaceBeg() + kSpaceSize); return ®ions[class_id]; } - uptr GetMetadataEnd(uptr region_beg) { + uptr GetMetadataEnd(uptr region_beg) const { return region_beg + kRegionSize - kFreeArraySize; } - uptr GetChunkIdx(uptr chunk, uptr size) { + uptr GetChunkIdx(uptr chunk, uptr size) const { if (!kUsingConstantSpaceBeg) chunk -= SpaceBeg(); @@ -382,13 +622,12 @@ class SizeClassAllocator64 { return (u32)offset / (u32)size; } - CompactPtrT *GetFreeArray(uptr region_beg) { - return reinterpret_cast<CompactPtrT *>(region_beg + kRegionSize - - kFreeArraySize); + CompactPtrT *GetFreeArray(uptr region_beg) const { + return reinterpret_cast<CompactPtrT *>(GetMetadataEnd(region_beg)); } bool MapWithCallback(uptr beg, uptr size) { - uptr mapped = reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(beg, size)); + uptr mapped = address_range.Map(beg, size); if (UNLIKELY(!mapped)) return false; CHECK_EQ(beg, mapped); @@ -397,21 +636,21 @@ class SizeClassAllocator64 { } void MapWithCallbackOrDie(uptr beg, uptr size) { - CHECK_EQ(beg, reinterpret_cast<uptr>(MmapFixedOrDie(beg, size))); + CHECK_EQ(beg, address_range.MapOrDie(beg, size)); MapUnmapCallback().OnMap(beg, size); } void UnmapWithCallbackOrDie(uptr beg, uptr size) { MapUnmapCallback().OnUnmap(beg, size); - UnmapOrDie(reinterpret_cast<void *>(beg), size); + address_range.Unmap(beg, size); } bool EnsureFreeArraySpace(RegionInfo *region, uptr region_beg, uptr num_freed_chunks) { uptr needed_space = num_freed_chunks * sizeof(CompactPtrT); if (region->mapped_free_array < needed_space) { - CHECK_LE(needed_space, kFreeArraySize); uptr new_mapped_free_array = RoundUpTo(needed_space, kFreeArrayMapSize); + CHECK_LE(new_mapped_free_array, kFreeArraySize); uptr current_map_end = reinterpret_cast<uptr>(GetFreeArray(region_beg)) + region->mapped_free_array; uptr new_map_size = new_mapped_free_array - region->mapped_free_array; @@ -422,55 +661,74 @@ class SizeClassAllocator64 { return true; } + // Check whether this size class is exhausted. + bool IsRegionExhausted(RegionInfo *region, uptr class_id, + uptr additional_map_size) { + if (LIKELY(region->mapped_user + region->mapped_meta + + additional_map_size <= kRegionSize - kFreeArraySize)) + return false; + if (!region->exhausted) { + region->exhausted = true; + Printf("%s: Out of memory. ", SanitizerToolName); + Printf("The process has exhausted %zuMB for size class %zu.\n", + kRegionSize >> 20, ClassIdToSize(class_id)); + } + return true; + } + NOINLINE bool PopulateFreeArray(AllocatorStats *stat, uptr class_id, RegionInfo *region, uptr requested_count) { // region->mutex is held. - const uptr size = ClassIdToSize(class_id); - const uptr new_space_beg = region->allocated_user; - const uptr new_space_end = new_space_beg + requested_count * size; const uptr region_beg = GetRegionBeginBySizeClass(class_id); + const uptr size = ClassIdToSize(class_id); + const uptr total_user_bytes = + region->allocated_user + requested_count * size; // Map more space for chunks, if necessary. - if (new_space_end > region->mapped_user) { - if (!kUsingConstantSpaceBeg && region->mapped_user == 0) - region->rand_state = static_cast<u32>(region_beg >> 12); // From ASLR. + if (LIKELY(total_user_bytes > region->mapped_user)) { + if (UNLIKELY(region->mapped_user == 0)) { + if (!kUsingConstantSpaceBeg && kRandomShuffleChunks) + // The random state is initialized from ASLR. + region->rand_state = static_cast<u32>(region_beg >> 12); + // Postpone the first release to OS attempt for ReleaseToOSIntervalMs, + // preventing just allocated memory from being released sooner than + // necessary and also preventing extraneous ReleaseMemoryPagesToOS calls + // for short lived processes. + // Do it only when the feature is turned on, to avoid a potentially + // extraneous syscall. + if (ReleaseToOSIntervalMs() >= 0) + region->rtoi.last_release_at_ns = MonotonicNanoTime(); + } // Do the mmap for the user memory. - uptr map_size = kUserMapSize; - while (new_space_end > region->mapped_user + map_size) - map_size += kUserMapSize; - CHECK_GE(region->mapped_user + map_size, new_space_end); + const uptr user_map_size = + RoundUpTo(total_user_bytes - region->mapped_user, kUserMapSize); + if (UNLIKELY(IsRegionExhausted(region, class_id, user_map_size))) + return false; if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user, - map_size))) + user_map_size))) return false; - stat->Add(AllocatorStatMapped, map_size); - region->mapped_user += map_size; + stat->Add(AllocatorStatMapped, user_map_size); + region->mapped_user += user_map_size; } - const uptr new_chunks_count = (region->mapped_user - new_space_beg) / size; + const uptr new_chunks_count = + (region->mapped_user - region->allocated_user) / size; - // Calculate the required space for metadata. - const uptr requested_allocated_meta = - region->allocated_meta + new_chunks_count * kMetadataSize; - uptr requested_mapped_meta = region->mapped_meta; - while (requested_allocated_meta > requested_mapped_meta) - requested_mapped_meta += kMetaMapSize; - // Check whether this size class is exhausted. - if (region->mapped_user + requested_mapped_meta > - kRegionSize - kFreeArraySize) { - if (!region->exhausted) { - region->exhausted = true; - Printf("%s: Out of memory. ", SanitizerToolName); - Printf("The process has exhausted %zuMB for size class %zu.\n", - kRegionSize >> 20, size); + if (kMetadataSize) { + // Calculate the required space for metadata. + const uptr total_meta_bytes = + region->allocated_meta + new_chunks_count * kMetadataSize; + const uptr meta_map_size = (total_meta_bytes > region->mapped_meta) ? + RoundUpTo(total_meta_bytes - region->mapped_meta, kMetaMapSize) : 0; + // Map more space for metadata, if necessary. + if (meta_map_size) { + if (UNLIKELY(IsRegionExhausted(region, class_id, meta_map_size))) + return false; + if (UNLIKELY(!MapWithCallback( + GetMetadataEnd(region_beg) - region->mapped_meta - meta_map_size, + meta_map_size))) + return false; + region->mapped_meta += meta_map_size; } - return false; - } - // Map more space for metadata, if necessary. - if (requested_mapped_meta > region->mapped_meta) { - if (UNLIKELY(!MapWithCallback( - GetMetadataEnd(region_beg) - requested_mapped_meta, - requested_mapped_meta - region->mapped_meta))) - return false; - region->mapped_meta = requested_mapped_meta; } // If necessary, allocate more space for the free array and populate it with @@ -479,7 +737,7 @@ class SizeClassAllocator64 { if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg, total_freed_chunks))) return false; CompactPtrT *free_array = GetFreeArray(region_beg); - for (uptr i = 0, chunk = new_space_beg; i < new_chunks_count; + for (uptr i = 0, chunk = region->allocated_user; i < new_chunks_count; i++, chunk += size) free_array[total_freed_chunks - 1 - i] = PointerToCompactPtr(0, chunk); if (kRandomShuffleChunks) @@ -491,27 +749,66 @@ class SizeClassAllocator64 { region->num_freed_chunks += new_chunks_count; region->allocated_user += new_chunks_count * size; CHECK_LE(region->allocated_user, region->mapped_user); - region->allocated_meta = requested_allocated_meta; + region->allocated_meta += new_chunks_count * kMetadataSize; CHECK_LE(region->allocated_meta, region->mapped_meta); region->exhausted = false; + // TODO(alekseyshl): Consider bumping last_release_at_ns here to prevent + // MaybeReleaseToOS from releasing just allocated pages or protect these + // not yet used chunks some other way. + return true; } - void MaybeReleaseChunkRange(uptr region_beg, uptr chunk_size, - CompactPtrT first, CompactPtrT last) { - uptr beg_ptr = CompactPtrToPointer(region_beg, first); - uptr end_ptr = CompactPtrToPointer(region_beg, last) + chunk_size; - ReleaseMemoryPagesToOS(beg_ptr, end_ptr); - } + class MemoryMapper { + public: + MemoryMapper(const ThisT& base_allocator, uptr class_id) + : allocator(base_allocator), + region_base(base_allocator.GetRegionBeginBySizeClass(class_id)), + released_ranges_count(0), + released_bytes(0) { + } + + uptr GetReleasedRangesCount() const { + return released_ranges_count; + } + + uptr GetReleasedBytes() const { + return released_bytes; + } + + uptr MapPackedCounterArrayBuffer(uptr buffer_size) { + // TODO(alekseyshl): The idea to explore is to check if we have enough + // space between num_freed_chunks*sizeof(CompactPtrT) and + // mapped_free_array to fit buffer_size bytes and use that space instead + // of mapping a temporary one. + return reinterpret_cast<uptr>( + MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters")); + } - // Attempts to release some RAM back to OS. The region is expected to be - // locked. - // Algorithm: - // * Sort the chunks. - // * Find ranges fully covered by free-d chunks - // * Release them to OS with madvise. - void MaybeReleaseToOS(uptr class_id) { + void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) { + UnmapOrDie(reinterpret_cast<void *>(buffer), buffer_size); + } + + // Releases [from, to) range of pages back to OS. + void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) { + const uptr from_page = allocator.CompactPtrToPointer(region_base, from); + const uptr to_page = allocator.CompactPtrToPointer(region_base, to); + ReleaseMemoryPagesToOS(from_page, to_page); + released_ranges_count++; + released_bytes += to_page - from_page; + } + + private: + const ThisT& allocator; + const uptr region_base; + uptr released_ranges_count; + uptr released_bytes; + }; + + // Attempts to release RAM occupied by freed chunks back to OS. The region is + // expected to be locked. + void MaybeReleaseToOS(uptr class_id, bool force) { RegionInfo *region = GetRegionInfo(class_id); const uptr chunk_size = ClassIdToSize(class_id); const uptr page_size = GetPageSizeCached(); @@ -524,37 +821,29 @@ class SizeClassAllocator64 { return; // Nothing new to release. } - s32 interval_ms = ReleaseToOSIntervalMs(); - if (interval_ms < 0) - return; + if (!force) { + s32 interval_ms = ReleaseToOSIntervalMs(); + if (interval_ms < 0) + return; - u64 now_ns = NanoTime(); - if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > now_ns) - return; // Memory was returned recently. - region->rtoi.last_release_at_ns = now_ns; + if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > + MonotonicNanoTime()) { + return; // Memory was returned recently. + } + } - uptr region_beg = GetRegionBeginBySizeClass(class_id); - CompactPtrT *free_array = GetFreeArray(region_beg); - SortArray(free_array, n); + MemoryMapper memory_mapper(*this, class_id); - const uptr scaled_chunk_size = chunk_size >> kCompactPtrScale; - const uptr kScaledGranularity = page_size >> kCompactPtrScale; + ReleaseFreeMemoryToOS<MemoryMapper>( + GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size, + RoundUpTo(region->allocated_user, page_size) / page_size, + &memory_mapper); - uptr range_beg = free_array[0]; - uptr prev = free_array[0]; - for (uptr i = 1; i < n; i++) { - uptr chunk = free_array[i]; - CHECK_GT(chunk, prev); - if (chunk - prev != scaled_chunk_size) { - CHECK_GT(chunk - prev, scaled_chunk_size); - if (prev + scaled_chunk_size - range_beg >= kScaledGranularity) { - MaybeReleaseChunkRange(region_beg, chunk_size, range_beg, prev); - region->rtoi.n_freed_at_last_release = region->stats.n_freed; - region->rtoi.num_releases++; - } - range_beg = chunk; - } - prev = chunk; + if (memory_mapper.GetReleasedRangesCount() > 0) { + region->rtoi.n_freed_at_last_release = region->stats.n_freed; + region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount(); + region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes(); } + region->rtoi.last_release_at_ns = MonotonicNanoTime(); } }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h index 7151a4636056..2bd83b2eb5a4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h @@ -134,8 +134,9 @@ class SizeClassMap { static const uptr kMaxSize = 1UL << kMaxSizeLog; static const uptr kNumClasses = - kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1; + kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1 + 1; static const uptr kLargestClassID = kNumClasses - 2; + static const uptr kBatchClassID = kNumClasses - 1; COMPILER_CHECK(kNumClasses >= 16 && kNumClasses <= 256); static const uptr kNumClassesRounded = kNumClasses <= 32 ? 32 : @@ -143,6 +144,11 @@ class SizeClassMap { kNumClasses <= 128 ? 128 : 256; static uptr Size(uptr class_id) { + // Estimate the result for kBatchClassID because this class does not know + // the exact size of TransferBatch. It's OK since we are using the actual + // sizeof(TransferBatch) where it matters. + if (UNLIKELY(class_id == kBatchClassID)) + return kMaxNumCachedHint * sizeof(uptr); if (class_id <= kMidClass) return kMinSize * class_id; class_id -= kMidClass; @@ -151,9 +157,10 @@ class SizeClassMap { } static uptr ClassID(uptr size) { + if (UNLIKELY(size > kMaxSize)) + return 0; if (size <= kMidSize) return (size + kMinSize - 1) >> kMinSizeLog; - if (size > kMaxSize) return 0; uptr l = MostSignificantSetBitIndex(size); uptr hbits = (size >> (l - S)) & M; uptr lbits = size & ((1 << (l - S)) - 1); @@ -162,7 +169,13 @@ class SizeClassMap { } static uptr MaxCachedHint(uptr class_id) { - if (class_id == 0) return 0; + // Estimate the result for kBatchClassID because this class does not know + // the exact size of TransferBatch. We need to cache fewer batches than user + // chunks, so this number can be small. + if (UNLIKELY(class_id == kBatchClassID)) + return 16; + if (UNLIKELY(class_id == 0)) + return 0; uptr n = (1UL << kMaxBytesCachedLog) / Size(class_id); return Max<uptr>(1, Min(kMaxNumCachedHint, n)); } @@ -178,6 +191,8 @@ class SizeClassMap { uptr p = prev_s ? (d * 100 / prev_s) : 0; uptr l = s ? MostSignificantSetBitIndex(s) : 0; uptr cached = MaxCachedHint(i) * s; + if (i == kBatchClassID) + d = p = l = 0; Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd " "cached: %zd %zd; id %zd\n", i, Size(i), d, p, l, MaxCachedHint(i), cached, ClassID(s)); @@ -192,12 +207,13 @@ class SizeClassMap { // Printf("Validate: c%zd\n", c); uptr s = Size(c); CHECK_NE(s, 0U); + if (c == kBatchClassID) + continue; CHECK_EQ(ClassID(s), c); - if (c != kNumClasses - 1) + if (c < kLargestClassID) CHECK_EQ(ClassID(s + 1), c + 1); CHECK_EQ(ClassID(s - 1), c); - if (c) - CHECK_GT(Size(c), Size(c-1)); + CHECK_GT(Size(c), Size(c - 1)); } CHECK_EQ(ClassID(kMaxSize + 1), 0); @@ -207,7 +223,7 @@ class SizeClassMap { CHECK_LT(c, kNumClasses); CHECK_GE(Size(c), s); if (c > 0) - CHECK_LT(Size(c-1), s); + CHECK_LT(Size(c - 1), s); } } }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h index 47c2b12a2049..76a0bf71425d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h @@ -47,12 +47,12 @@ # define ASM_HIDDEN(symbol) .hidden symbol # define ASM_TYPE_FUNCTION(symbol) .type symbol, @function # define ASM_SIZE(symbol) .size symbol, .-symbol -# define ASM_TSAN_SYMBOL(symbol) symbol -# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) symbol +# define ASM_SYMBOL(symbol) symbol +# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol #else # define ASM_HIDDEN(symbol) # define ASM_TYPE_FUNCTION(symbol) # define ASM_SIZE(symbol) -# define ASM_TSAN_SYMBOL(symbol) _##symbol -# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol +# define ASM_SYMBOL(symbol) _##symbol +# define ASM_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h index d8472732ff21..0f65814c398f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h @@ -22,7 +22,7 @@ namespace __sanitizer { template <class basic_int_t = uptr> class BasicBitVector { public: - enum SizeEnum { kSize = sizeof(basic_int_t) * 8 }; + enum SizeEnum : uptr { kSize = sizeof(basic_int_t) * 8 }; uptr size() const { return kSize; } // No CTOR. @@ -115,7 +115,7 @@ class TwoLevelBitVector { // This structure allows O(kLevel1Size) time for clear() and empty(), // as well fast handling of sparse BVs. public: - enum SizeEnum { kSize = BV::kSize * BV::kSize * kLevel1Size }; + enum SizeEnum : uptr { kSize = BV::kSize * BV::kSize * kLevel1Size }; // No CTOR. uptr size() const { return kSize; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h index df72f1c2d471..9c2db9ae4954 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h @@ -25,7 +25,7 @@ namespace __sanitizer { template<class BV> class BVGraph { public: - enum SizeEnum { kSize = BV::kSize }; + enum SizeEnum : uptr { kSize = BV::kSize }; uptr size() const { return kSize; } // No CTOR. void clear() { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc index 7e6f8fce76d2..ae26d5efc24f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc @@ -26,72 +26,7 @@ const char *SanitizerToolName = "SanitizerTool"; atomic_uint32_t current_verbosity; uptr PageSizeCached; - -StaticSpinMutex report_file_mu; -ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0}; - -void RawWrite(const char *buffer) { - report_file.Write(buffer, internal_strlen(buffer)); -} - -void ReportFile::ReopenIfNecessary() { - mu->CheckLocked(); - if (fd == kStdoutFd || fd == kStderrFd) return; - - uptr pid = internal_getpid(); - // If in tracer, use the parent's file. - if (pid == stoptheworld_tracer_pid) - pid = stoptheworld_tracer_ppid; - if (fd != kInvalidFd) { - // If the report file is already opened by the current process, - // do nothing. Otherwise the report file was opened by the parent - // process, close it now. - if (fd_pid == pid) - return; - else - CloseFile(fd); - } - - const char *exe_name = GetProcessName(); - if (common_flags()->log_exe_name && exe_name) { - internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix, - exe_name, pid); - } else { - internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid); - } - fd = OpenFile(full_path, WrOnly); - if (fd == kInvalidFd) { - const char *ErrorMsgPrefix = "ERROR: Can't open file: "; - WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix)); - WriteToFile(kStderrFd, full_path, internal_strlen(full_path)); - Die(); - } - fd_pid = pid; -} - -void ReportFile::SetReportPath(const char *path) { - if (!path) - return; - uptr len = internal_strlen(path); - if (len > sizeof(path_prefix) - 100) { - Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n", - path[0], path[1], path[2], path[3], - path[4], path[5], path[6], path[7]); - Die(); - } - - SpinMutexLock l(mu); - if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd) - CloseFile(fd); - fd = kInvalidFd; - if (internal_strcmp(path, "stdout") == 0) { - fd = kStdoutFd; - } else if (internal_strcmp(path, "stderr") == 0) { - fd = kStderrFd; - } else { - internal_snprintf(path_prefix, kMaxPathLength, "%s", path); - } -} +u32 NumberOfCPUsCached; // PID of the tracer task in StopTheWorld. It shares the address space with the // main process, but has a different PID and thus requires special handling. @@ -120,42 +55,6 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, UNREACHABLE("unable to mmap"); } -bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, - uptr *read_len, uptr max_len, error_t *errno_p) { - uptr PageSize = GetPageSizeCached(); - uptr kMinFileLen = PageSize; - *buff = nullptr; - *buff_size = 0; - *read_len = 0; - // The files we usually open are not seekable, so try different buffer sizes. - for (uptr size = kMinFileLen; size <= max_len; size *= 2) { - fd_t fd = OpenFile(file_name, RdOnly, errno_p); - if (fd == kInvalidFd) return false; - UnmapOrDie(*buff, *buff_size); - *buff = (char*)MmapOrDie(size, __func__); - *buff_size = size; - *read_len = 0; - // Read up to one page at a time. - bool reached_eof = false; - while (*read_len + PageSize <= size) { - uptr just_read; - if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) { - UnmapOrDie(*buff, *buff_size); - return false; - } - if (just_read == 0) { - reached_eof = true; - break; - } - *read_len += just_read; - } - CloseFile(fd); - if (reached_eof) // We've read the whole file. - break; - } - return true; -} - typedef bool UptrComparisonFunction(const uptr &a, const uptr &b); typedef bool U32ComparisonFunction(const u32 &a, const u32 &b); @@ -285,9 +184,10 @@ void LoadedModule::clear() { } void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable, - bool writable) { + bool writable, const char *name) { void *mem = InternalAlloc(sizeof(AddressRange)); - AddressRange *r = new(mem) AddressRange(beg, end, executable, writable); + AddressRange *r = + new(mem) AddressRange(beg, end, executable, writable, name); ranges_.push_back(r); if (executable && end > max_executable_address_) max_executable_address_ = end; @@ -359,36 +259,6 @@ bool TemplateMatch(const char *templ, const char *str) { return true; } -static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':'; - -char *FindPathToBinary(const char *name) { - if (FileExists(name)) { - return internal_strdup(name); - } - - const char *path = GetEnv("PATH"); - if (!path) - return nullptr; - uptr name_len = internal_strlen(name); - InternalScopedBuffer<char> buffer(kMaxPathLength); - const char *beg = path; - while (true) { - const char *end = internal_strchrnul(beg, kPathSeparator); - uptr prefix_len = end - beg; - if (prefix_len + name_len + 2 <= kMaxPathLength) { - internal_memcpy(buffer.data(), beg, prefix_len); - buffer[prefix_len] = '/'; - internal_memcpy(&buffer[prefix_len + 1], name, name_len); - buffer[prefix_len + 1 + name_len] = '\0'; - if (FileExists(buffer.data())) - return internal_strdup(buffer.data()); - } - if (*end == '\0') break; - beg = end + 1; - } - return nullptr; -} - static char binary_name_cache_str[kMaxPathLength]; static char process_name_cache_str[kMaxPathLength]; @@ -482,15 +352,6 @@ static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr), using namespace __sanitizer; // NOLINT extern "C" { -void __sanitizer_set_report_path(const char *path) { - report_file.SetReportPath(path); -} - -void __sanitizer_set_report_fd(void *fd) { - report_file.fd = (fd_t)reinterpret_cast<uptr>(fd); - report_file.fd_pid = internal_getpid(); -} - SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_report_error_summary, const char *error_summary) { Printf("%s\n", error_summary); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 89aae579856a..1fbaee7e39a1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -29,8 +29,11 @@ extern "C" void _ReadWriteBarrier(); #endif namespace __sanitizer { -struct StackTrace; + struct AddressInfo; +struct BufferedStackTrace; +struct SignalContext; +struct StackTrace; // Constants. const uptr kWordSize = SANITIZER_WORDSIZE / 8; @@ -71,6 +74,7 @@ INLINE uptr GetPageSizeCached() { } uptr GetMmapGranularity(); uptr GetMaxVirtualAddress(); +uptr GetMaxUserVirtualAddress(); // Threads tid_t GetTid(); uptr GetThreadSelf(); @@ -125,6 +129,30 @@ void CheckVMASize(); void RunMallocHooks(const void *ptr, uptr size); void RunFreeHooks(const void *ptr); +class ReservedAddressRange { + public: + uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0); + uptr Map(uptr fixed_addr, uptr size); + uptr MapOrDie(uptr fixed_addr, uptr size); + void Unmap(uptr addr, uptr size); + void *base() const { return base_; } + uptr size() const { return size_; } + + private: + void* base_; + uptr size_; + const char* name_; + uptr os_handle_; +}; + +typedef void (*fill_profile_f)(uptr start, uptr rss, bool file, + /*out*/uptr *stats, uptr stats_size); + +// Parse the contents of /proc/self/smaps and generate a memory profile. +// |cb| is a tool-specific callback that fills the |stats| array containing +// |stats_size| elements. +void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size); + // InternalScopedBuffer can be used instead of large stack arrays to // keep frame size low. // FIXME: use InternalAlloc instead of MmapOrDie once @@ -179,12 +207,15 @@ class LowLevelAllocator { char *allocated_end_; char *allocated_current_; }; +// Set the min alignment of LowLevelAllocator to at least alignment. +void SetLowLevelAllocateMinAlignment(uptr alignment); typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size); // Allows to register tool-specific callbacks for LowLevelAllocator. // Passing NULL removes the callback. void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback); // IO +void CatastrophicErrorWrite(const char *buffer, uptr length); void RawWrite(const char *buffer); bool ColorizeReports(); void RemoveANSIEscapeSequencesFromString(char *buffer); @@ -200,67 +231,18 @@ void SetPrintfAndReportCallback(void (*callback)(const char *)); if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \ } while (0) -// Can be used to prevent mixing error reports from different sanitizers. -extern StaticSpinMutex CommonSanitizerReportMutex; - -struct ReportFile { - void Write(const char *buffer, uptr length); - bool SupportsColors(); - void SetReportPath(const char *path); - - // Don't use fields directly. They are only declared public to allow - // aggregate initialization. - - // Protects fields below. - StaticSpinMutex *mu; - // Opened file descriptor. Defaults to stderr. It may be equal to - // kInvalidFd, in which case new file will be opened when necessary. - fd_t fd; - // Path prefix of report file, set via __sanitizer_set_report_path. - char path_prefix[kMaxPathLength]; - // Full path to report, obtained as <path_prefix>.PID - char full_path[kMaxPathLength]; - // PID of the process that opened fd. If a fork() occurs, - // the PID of child will be different from fd_pid. - uptr fd_pid; +// Lock sanitizer error reporting and protects against nested errors. +class ScopedErrorReportLock { + public: + ScopedErrorReportLock(); + ~ScopedErrorReportLock(); - private: - void ReopenIfNecessary(); + static void CheckLocked(); }; -extern ReportFile report_file; extern uptr stoptheworld_tracer_pid; extern uptr stoptheworld_tracer_ppid; -enum FileAccessMode { - RdOnly, - WrOnly, - RdWr -}; - -// Returns kInvalidFd on error. -fd_t OpenFile(const char *filename, FileAccessMode mode, - error_t *errno_p = nullptr); -void CloseFile(fd_t); - -// Return true on success, false on error. -bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, - uptr *bytes_read = nullptr, error_t *error_p = nullptr); -bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, - uptr *bytes_written = nullptr, error_t *error_p = nullptr); - -bool RenameFile(const char *oldpath, const char *newpath, - error_t *error_p = nullptr); - -// Scoped file handle closer. -struct FileCloser { - explicit FileCloser(fd_t fd) : fd(fd) {} - ~FileCloser() { CloseFile(fd); } - fd_t fd; -}; - -bool SupportsColoredOutput(fd_t fd); - // Opens the file 'file_name" and reads up to 'max_len' bytes. // The resulting buffer is mmaped and stored in '*buff'. // The size of the mmaped region is stored in '*buff_size'. @@ -269,11 +251,6 @@ bool SupportsColoredOutput(fd_t fd); bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, uptr *read_len, uptr max_len = 1 << 26, error_t *errno_p = nullptr); -// Maps given file to virtual memory, and returns pointer to it -// (or NULL if mapping fails). Stores the size of mmaped region -// in '*buff_size'. -void *MapFileToMemory(const char *file_name, uptr *buff_size); -void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset); bool IsAccessibleMemoryRange(uptr beg, uptr size); @@ -293,27 +270,8 @@ void CacheBinaryName(); void DisableCoreDumperIfNecessary(); void DumpProcessMap(); void PrintModuleMap(); -bool FileExists(const char *filename); const char *GetEnv(const char *name); bool SetEnv(const char *name, const char *value); -const char *GetPwd(); -char *FindPathToBinary(const char *name); -bool IsPathSeparator(const char c); -bool IsAbsolutePath(const char *path); -// Starts a subprocess and returs its pid. -// If *_fd parameters are not kInvalidFd their corresponding input/output -// streams will be redirect to the file. The files will always be closed -// in parent process even in case of an error. -// The child process will close all fds after STDERR_FILENO -// before passing control to a program. -pid_t StartSubprocess(const char *filename, const char *const argv[], - fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd, - fd_t stderr_fd = kInvalidFd); -// Checks if specified process is still running -bool IsProcessRunning(pid_t pid); -// Waits for the process to finish and returns its exit code. -// Returns -1 in case of an error. -int WaitForProcess(pid_t pid); u32 GetUid(); void ReExec(); @@ -337,6 +295,7 @@ uptr GetTlsSize(); void SleepForSeconds(int seconds); void SleepForMillis(int millis); u64 NanoTime(); +u64 MonotonicNanoTime(); int Atexit(void (*function)(void)); void SortArray(uptr *array, uptr size); void SortArray(u32 *array, uptr size); @@ -385,7 +344,24 @@ void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)); typedef void (*SignalHandlerType)(int, void *, void *); HandleSignalMode GetHandleSignalMode(int signum); void InstallDeadlySignalHandlers(SignalHandlerType handler); -const char *DescribeSignalOrException(int signo); + +// Signal reporting. +// Each sanitizer uses slightly different implementation of stack unwinding. +typedef void (*UnwindSignalStackCallbackType)(const SignalContext &sig, + const void *callback_context, + BufferedStackTrace *stack); +// Print deadly signal report and die. +void HandleDeadlySignal(void *siginfo, void *context, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context); + +// Part of HandleDeadlySignal, exposed for asan. +void StartReportDeadlySignal(); +// Part of HandleDeadlySignal, exposed for asan. +void ReportDeadlySignal(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context); + // Alternative signal stack (POSIX-only). void SetAlternateSignalStack(); void UnsetAlternateSignalStack(); @@ -702,6 +678,7 @@ inline const char *ModuleArchToString(ModuleArch arch) { } const uptr kModuleUUIDSize = 16; +const uptr kMaxSegName = 16; // Represents a binary loaded into virtual memory (e.g. this can be an // executable or a shared object). @@ -720,7 +697,8 @@ class LoadedModule { void set(const char *module_name, uptr base_address, ModuleArch arch, u8 uuid[kModuleUUIDSize], bool instrumented); void clear(); - void addAddressRange(uptr beg, uptr end, bool executable, bool writable); + void addAddressRange(uptr beg, uptr end, bool executable, bool writable, + const char *name = nullptr); bool containsAddress(uptr address) const; const char *full_name() const { return full_name_; } @@ -736,13 +714,17 @@ class LoadedModule { uptr end; bool executable; bool writable; + char name[kMaxSegName]; - AddressRange(uptr beg, uptr end, bool executable, bool writable) + AddressRange(uptr beg, uptr end, bool executable, bool writable, + const char *name) : next(nullptr), beg(beg), end(end), executable(executable), - writable(writable) {} + writable(writable) { + internal_strncpy(this->name, (name ? name : ""), ARRAY_SIZE(this->name)); + } }; const IntrusiveList<AddressRange> &ranges() const { return ranges_; } @@ -761,9 +743,10 @@ class LoadedModule { // filling this information. class ListOfModules { public: - ListOfModules() : modules_(kInitialCapacity) {} + ListOfModules() : initialized(false) {} ~ListOfModules() { clear(); } void init(); + void fallbackInit(); // Uses fallback init if available, otherwise clears const LoadedModule *begin() const { return modules_.begin(); } LoadedModule *begin() { return modules_.begin(); } const LoadedModule *end() const { return modules_.end(); } @@ -779,10 +762,15 @@ class ListOfModules { for (auto &module : modules_) module.clear(); modules_.clear(); } + void clearOrInit() { + initialized ? clear() : modules_.Initialize(kInitialCapacity); + initialized = true; + } - InternalMmapVector<LoadedModule> modules_; + InternalMmapVectorNoCtor<LoadedModule> modules_; // We rarely have more than 16K loaded modules. static const uptr kInitialCapacity = 1 << 14; + bool initialized; }; // Callback type for iterating over a set of memory ranges. @@ -858,35 +846,49 @@ static inline void SanitizerBreakOptimization(void *arg) { } struct SignalContext { + void *siginfo; void *context; uptr addr; uptr pc; uptr sp; uptr bp; bool is_memory_access; - enum WriteFlag { UNKNOWN, READ, WRITE } write_flag; - SignalContext(void *context, uptr addr, uptr pc, uptr sp, uptr bp, - bool is_memory_access, WriteFlag write_flag) - : context(context), - addr(addr), - pc(pc), - sp(sp), - bp(bp), - is_memory_access(is_memory_access), - write_flag(write_flag) {} + // VS2013 doesn't implement unrestricted unions, so we need a trivial default + // constructor + SignalContext() = default; + + // Creates signal context in a platform-specific manner. + // SignalContext is going to keep pointers to siginfo and context without + // owning them. + SignalContext(void *siginfo, void *context) + : siginfo(siginfo), + context(context), + addr(GetAddress()), + is_memory_access(IsMemoryAccess()), + write_flag(GetWriteFlag()) { + InitPcSpBp(); + } static void DumpAllRegisters(void *context); - // Creates signal context in a platform-specific manner. - static SignalContext Create(void *siginfo, void *context); + // Type of signal e.g. SIGSEGV or EXCEPTION_ACCESS_VIOLATION. + int GetType() const; - // Returns true if the "context" indicates a memory write. - static WriteFlag GetWriteFlag(void *context); -}; + // String description of the signal. + const char *Describe() const; -void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp); + // Returns true if signal is stack overflow. + bool IsStackOverflow() const; + + private: + // Platform specific initialization. + void InitPcSpBp(); + uptr GetAddress() const; + WriteFlag GetWriteFlag() const; + bool IsMemoryAccess() const; +}; void MaybeReexec(); @@ -929,8 +931,17 @@ const s32 kReleaseToOSIntervalNever = -1; void CheckNoDeepBind(const char *filename, int flag); // Returns the requested amount of random data (up to 256 bytes) that can then -// be used to seed a PRNG. -bool GetRandom(void *buffer, uptr length); +// be used to seed a PRNG. Defaults to blocking like the underlying syscall. +bool GetRandom(void *buffer, uptr length, bool blocking = true); + +// Returns the number of logical processors on the system. +u32 GetNumberOfCPUs(); +extern u32 NumberOfCPUsCached; +INLINE u32 GetNumberOfCPUsCached() { + if (!NumberOfCPUsCached) + NumberOfCPUsCached = GetNumberOfCPUs(); + return NumberOfCPUsCached; +} } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index 8607bf44902d..592a8c7b032a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -43,6 +43,7 @@ #include "sanitizer_errno.h" #include "sanitizer_placement_new.h" #include "sanitizer_platform_interceptors.h" +#include "sanitizer_symbolizer.h" #include "sanitizer_tls_get_addr.h" #include <stdarg.h> @@ -68,6 +69,46 @@ #define iconv __bsd_iconv #endif +#if SANITIZER_NETBSD +#define clock_getres __clock_getres50 +#define clock_gettime __clock_gettime50 +#define clock_settime __clock_settime50 +#define ctime __ctime50 +#define ctime_r __ctime_r50 +#define getitimer __getitimer50 +#define getpwent __getpwent50 +#define getpwnam __getpwnam50 +#define getpwnam_r __getpwnam_r50 +#define getpwuid __getpwuid50 +#define getpwuid_r __getpwuid_r50 +#define getutent __getutent50 +#define getutxent __getutxent50 +#define getutxid __getutxid50 +#define getutxline __getutxline50 +#define glob __glob30 +#define gmtime __gmtime50 +#define gmtime_r __gmtime_r50 +#define localtime_r __localtime_r50 +#define mktime __mktime50 +#define opendir __opendir30 +#define readdir __readdir30 +#define readdir_r __readdir_r30 +#define scandir __scandir30 +#define setitimer __setitimer50 +#define setlocale __setlocale50 +#define shmctl __shmctl50 +#define sigemptyset __sigemptyset14 +#define sigfillset __sigfillset14 +#define sigpending __sigpending14 +#define sigprocmask __sigprocmask14 +#define sigtimedwait __sigtimedwait50 +#define stat __stat50 +#define time __time50 +#define times __times13 +#define wait3 __wait350 +#define wait4 __wait450 +#endif + // Platform-specific options. #if SANITIZER_MAC namespace __sanitizer { @@ -258,7 +299,7 @@ typedef AddrHashMap<CommonInterceptorMetadata, 31051> MetadataHashMap; static MetadataHashMap *interceptor_metadata_map; -#if SI_NOT_WINDOWS +#if SI_POSIX UNUSED static void SetInterceptorMetadata(__sanitizer_FILE *addr, const FileMetadata &file) { MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr); @@ -272,7 +313,7 @@ UNUSED static const FileMetadata *GetInterceptorMetadata( MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr, /* remove */ false, /* create */ false); - if (h.exists()) { + if (addr && h.exists()) { CHECK(!h.created()); CHECK(h->type == CommonInterceptorMetadata::CIMT_FILE); return &h->file; @@ -285,7 +326,7 @@ UNUSED static void DeleteInterceptorMetadata(void *addr) { MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr, true); CHECK(h.exists()); } -#endif // SI_NOT_WINDOWS +#endif // SI_POSIX #if SANITIZER_INTERCEPT_STRLEN INTERCEPTOR(SIZE_T, strlen, const char *s) { @@ -886,7 +927,7 @@ INTERCEPTOR(long double, frexpl, long double x, int *exp) { #define INIT_FREXPF_FREXPL #endif // SANITIZER_INTERCEPT_FREXPF_FREXPL -#if SI_NOT_WINDOWS +#if SI_POSIX static void write_iovec(void *ctx, struct __sanitizer_iovec *iovec, SIZE_T iovlen, SIZE_T maxlen) { for (SIZE_T i = 0; i < iovlen && maxlen; ++i) { @@ -1176,12 +1217,14 @@ INTERCEPTOR(unsigned long, time, unsigned long *t) { #if SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS static void unpoison_tm(void *ctx, __sanitizer_tm *tm) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tm, sizeof(*tm)); +#if !SANITIZER_SOLARIS if (tm->tm_zone) { // Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone // can point to shared memory and tsan would report a data race. COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone, REAL(strlen(tm->tm_zone)) + 1); } +#endif } INTERCEPTOR(__sanitizer_tm *, localtime, unsigned long *timep) { void *ctx; @@ -1509,6 +1552,12 @@ INTERCEPTOR(int, vsnprintf, char *str, SIZE_T size, const char *format, va_list ap) VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __vsnprintf_chk, char *str, SIZE_T size, int flag, + SIZE_T size_to, const char *format, va_list ap) +VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap) +#endif + #if SANITIZER_INTERCEPT_PRINTF_L INTERCEPTOR(int, vsnprintf_l, char *str, SIZE_T size, void *loc, const char *format, va_list ap) @@ -1522,6 +1571,12 @@ FORMAT_INTERCEPTOR_IMPL(snprintf_l, vsnprintf_l, str, size, loc, format) INTERCEPTOR(int, vsprintf, char *str, const char *format, va_list ap) VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __vsprintf_chk, char *str, int flag, SIZE_T size_to, + const char *format, va_list ap) +VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap) +#endif + INTERCEPTOR(int, vasprintf, char **strp, const char *format, va_list ap) VASPRINTF_INTERCEPTOR_IMPL(vasprintf, strp, format, ap) @@ -1550,12 +1605,30 @@ FORMAT_INTERCEPTOR_IMPL(printf, vprintf, format) INTERCEPTOR(int, fprintf, __sanitizer_FILE *stream, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(fprintf, vfprintf, stream, format) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __fprintf_chk, __sanitizer_FILE *stream, SIZE_T size, + const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__fprintf_chk, vfprintf, stream, format) +#endif + INTERCEPTOR(int, sprintf, char *str, const char *format, ...) // NOLINT FORMAT_INTERCEPTOR_IMPL(sprintf, vsprintf, str, format) // NOLINT +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __sprintf_chk, char *str, int flag, SIZE_T size_to, + const char *format, ...) // NOLINT +FORMAT_INTERCEPTOR_IMPL(__sprintf_chk, vsprintf, str, format) // NOLINT +#endif + INTERCEPTOR(int, snprintf, char *str, SIZE_T size, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(snprintf, vsnprintf, str, size, format) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __snprintf_chk, char *str, SIZE_T size, int flag, + SIZE_T size_to, const char *format, ...) // NOLINT +FORMAT_INTERCEPTOR_IMPL(__snprintf_chk, vsnprintf, str, size, format) // NOLINT +#endif + INTERCEPTOR(int, asprintf, char **strp, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(asprintf, vasprintf, strp, format) @@ -1595,6 +1668,17 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size, #define INIT_PRINTF #endif +#if SANITIZER_INTERCEPT___PRINTF_CHK +#define INIT___PRINTF_CHK \ + COMMON_INTERCEPT_FUNCTION(__sprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__snprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__vsprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__vsnprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__fprintf_chk); +#else +#define INIT___PRINTF_CHK +#endif + #if SANITIZER_INTERCEPT_PRINTF_L #define INIT_PRINTF_L \ COMMON_INTERCEPT_FUNCTION(snprintf_l); \ @@ -1723,7 +1807,8 @@ static void unpoison_group(void *ctx, __sanitizer_group *grp) { INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name); - COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); + if (name) + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_passwd *res = REAL(getpwnam)(name); if (res) unpoison_passwd(ctx, res); return res; @@ -2003,6 +2088,13 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) { } return res; } +namespace __sanitizer { +extern "C" { +int real_clock_gettime(u32 clk_id, void *tp) { + return REAL(clock_gettime)(clk_id, tp); +} +} // extern "C" +} // namespace __sanitizer INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp); @@ -2064,6 +2156,18 @@ static void unpoison_glob_t(void *ctx, __sanitizer_glob_t *pglob) { } } +#if SANITIZER_SOLARIS +INTERCEPTOR(int, glob, const char *pattern, int flags, + int (*errfunc)(const char *epath, int eerrno), + __sanitizer_glob_t *pglob) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, glob, pattern, flags, errfunc, pglob); + COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0); + int res = REAL(glob)(pattern, flags, errfunc, pglob); + if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob); + return res; +} +#else static THREADLOCAL __sanitizer_glob_t *pglob_copy; static void wrapped_gl_closedir(void *dir) { @@ -2127,7 +2231,14 @@ INTERCEPTOR(int, glob, const char *pattern, int flags, if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob); return res; } +#endif // SANITIZER_SOLARIS +#define INIT_GLOB \ + COMMON_INTERCEPT_FUNCTION(glob); +#else // SANITIZER_INTERCEPT_GLOB +#define INIT_GLOB +#endif // SANITIZER_INTERCEPT_GLOB +#if SANITIZER_INTERCEPT_GLOB64 INTERCEPTOR(int, glob64, const char *pattern, int flags, int (*errfunc)(const char *epath, int eerrno), __sanitizer_glob_t *pglob) { @@ -2156,12 +2267,11 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags, if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob); return res; } -#define INIT_GLOB \ - COMMON_INTERCEPT_FUNCTION(glob); \ +#define INIT_GLOB64 \ COMMON_INTERCEPT_FUNCTION(glob64); -#else // SANITIZER_INTERCEPT_GLOB -#define INIT_GLOB -#endif // SANITIZER_INTERCEPT_GLOB +#else // SANITIZER_INTERCEPT_GLOB64 +#define INIT_GLOB64 +#endif // SANITIZER_INTERCEPT_GLOB64 #if SANITIZER_INTERCEPT_WAIT // According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version @@ -2466,7 +2576,15 @@ INTERCEPTOR(struct __sanitizer_hostent *, gethostent, int fake) { if (res) write_hostent(ctx, res); return res; } +#define INIT_GETHOSTBYNAME \ + COMMON_INTERCEPT_FUNCTION(gethostent); \ + COMMON_INTERCEPT_FUNCTION(gethostbyaddr); \ + COMMON_INTERCEPT_FUNCTION(gethostbyname); +#else +#define INIT_GETHOSTBYNAME +#endif // SANITIZER_INTERCEPT_GETHOSTBYNAME +#if SANITIZER_INTERCEPT_GETHOSTBYNAME2 INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname2, char *name, int af) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, gethostbyname2, name, af); @@ -2474,14 +2592,10 @@ INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname2, char *name, int af) { if (res) write_hostent(ctx, res); return res; } -#define INIT_GETHOSTBYNAME \ - COMMON_INTERCEPT_FUNCTION(gethostent); \ - COMMON_INTERCEPT_FUNCTION(gethostbyaddr); \ - COMMON_INTERCEPT_FUNCTION(gethostbyname); \ - COMMON_INTERCEPT_FUNCTION(gethostbyname2); +#define INIT_GETHOSTBYNAME2 COMMON_INTERCEPT_FUNCTION(gethostbyname2); #else -#define INIT_GETHOSTBYNAME -#endif +#define INIT_GETHOSTBYNAME2 +#endif // SANITIZER_INTERCEPT_GETHOSTBYNAME2 #if SANITIZER_INTERCEPT_GETHOSTBYNAME_R INTERCEPTOR(int, gethostbyname_r, char *name, struct __sanitizer_hostent *ret, @@ -3371,7 +3485,7 @@ INTERCEPTOR(char *, strerror, int errnum) { // * GNU version returns message pointer, which points to either buf or some // static storage. #if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \ - SANITIZER_MAC || SANITIZER_ANDROID + SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD // POSIX version. Spec is not clear on whether buf is NULL-terminated. // At least on OSX, buf contents are valid even when the call fails. INTERCEPTOR(int, strerror_r, int errnum, char *buf, SIZE_T buflen) { @@ -3589,7 +3703,7 @@ INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds, if (fds && nfds) read_pollfd(ctx, fds, nfds); if (timeout_ts) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout_ts, struct_timespec_sz); - // FIXME: read sigmask when all of sigemptyset, etc are intercepted. + if (sigmask) COMMON_INTERCEPTOR_READ_RANGE(ctx, sigmask, sizeof(*sigmask)); int res = COMMON_INTERCEPTOR_BLOCK_REAL(ppoll)(fds, nfds, timeout_ts, sigmask); if (fds && nfds) write_pollfd(ctx, fds, nfds); @@ -3630,7 +3744,7 @@ INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) { INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigwait, set, sig); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3647,7 +3761,7 @@ INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) { INTERCEPTOR(int, sigwaitinfo, __sanitizer_sigset_t *set, void *info) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigwaitinfo, set, info); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3666,7 +3780,7 @@ INTERCEPTOR(int, sigtimedwait, __sanitizer_sigset_t *set, void *info, void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigtimedwait, set, info, timeout); if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3729,7 +3843,7 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigprocmask, how, set, oldset); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3827,6 +3941,15 @@ INTERCEPTOR(int, pthread_mutex_unlock, void *m) { #define INIT_PTHREAD_MUTEX_UNLOCK #endif +#if SANITIZER_NETBSD +INTERCEPTOR(int, __libc_mutex_lock, void *m) \ + ALIAS(WRAPPER_NAME(pthread_mutex_lock)); +INTERCEPTOR(int, __libc_mutex_unlock, void *m) \ + ALIAS(WRAPPER_NAME(pthread_mutex_unlock)); +INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate) \ + ALIAS(WRAPPER_NAME(pthread_setcancelstate)); +#endif + #if SANITIZER_INTERCEPT_GETMNTENT || SANITIZER_INTERCEPT_GETMNTENT_R static void write_mntent(void *ctx, __sanitizer_mntent *mnt) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt)); @@ -4404,7 +4527,7 @@ INTERCEPTOR(char *, tempnam, char *dir, char *pfx) { #define INIT_TEMPNAM #endif -#if SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP +#if SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && !SANITIZER_NETBSD INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name); @@ -4413,6 +4536,17 @@ INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name) { return REAL(pthread_setname_np)(thread, name); } #define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np); +#elif SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && SANITIZER_NETBSD +INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name, void *arg) { + void *ctx; + char newname[32]; // PTHREAD_MAX_NAMELEN_NP=32 + COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name, arg); + COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); + internal_snprintf(newname, sizeof(newname), name, arg); + COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, newname); + return REAL(pthread_setname_np)(thread, name, arg); +} +#define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np); #else #define INIT_PTHREAD_SETNAME_NP #endif @@ -5575,6 +5709,7 @@ INTERCEPTOR(void*, dlopen, const char *filename, int flag) { if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0); COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag); void *res = REAL(dlopen)(filename, flag); + Symbolizer::GetOrInit()->InvalidateModuleList(); COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res); return res; } @@ -5583,6 +5718,7 @@ INTERCEPTOR(int, dlclose, void *handle) { void *ctx; COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlclose, handle); int res = REAL(dlclose)(handle); + Symbolizer::GetOrInit()->InvalidateModuleList(); COMMON_INTERCEPTOR_LIBRARY_UNLOADED(); return res; } @@ -5821,7 +5957,7 @@ INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcancelstate, state, oldstate); int res = REAL(pthread_setcancelstate)(state, oldstate); - if (res == 0) + if (res == 0 && oldstate != nullptr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldstate, sizeof(*oldstate)); return res; } @@ -5830,7 +5966,7 @@ INTERCEPTOR(int, pthread_setcanceltype, int type, int *oldtype) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcanceltype, type, oldtype); int res = REAL(pthread_setcanceltype)(type, oldtype); - if (res == 0) + if (res == 0 && oldtype != nullptr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldtype, sizeof(*oldtype)); return res; } @@ -6313,6 +6449,7 @@ static void InitializeCommonInterceptors() { INIT_GETITIMER; INIT_TIME; INIT_GLOB; + INIT_GLOB64; INIT_WAIT; INIT_WAIT4; INIT_INET; @@ -6321,6 +6458,7 @@ static void InitializeCommonInterceptors() { INIT_GETNAMEINFO; INIT_GETSOCKNAME; INIT_GETHOSTBYNAME; + INIT_GETHOSTBYNAME2; INIT_GETHOSTBYNAME_R; INIT_GETHOSTBYNAME2_R; INIT_GETHOSTBYADDR_R; @@ -6457,4 +6595,12 @@ static void InitializeCommonInterceptors() { INIT_GETLOADAVG; INIT_WCSLEN; INIT_WCSCAT; + +#if SANITIZER_NETBSD + COMMON_INTERCEPT_FUNCTION(__libc_mutex_lock); + COMMON_INTERCEPT_FUNCTION(__libc_mutex_unlock); + COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate); +#endif + + INIT___PRINTF_CHK; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc index 4ed9afedf84a..24e7548a597e 100755 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc @@ -57,7 +57,9 @@ static void ioctl_table_fill() { _(SIOCGIFCONF, CUSTOM, 0); _(SIOCGPGRP, WRITE, sizeof(int)); _(SIOCSPGRP, READ, sizeof(int)); +#if !SANITIZER_SOLARIS _(TIOCCONS, NONE, 0); +#endif _(TIOCEXCL, NONE, 0); _(TIOCGETD, WRITE, sizeof(int)); _(TIOCGPGRP, WRITE, pid_t_sz); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc index 550427c906a6..2568df3a5bff 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc @@ -34,6 +34,7 @@ INTERFACE_FUNCTION(__sanitizer_get_heap_size) INTERFACE_FUNCTION(__sanitizer_get_ownership) INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes) INTERFACE_FUNCTION(__sanitizer_install_malloc_and_free_hooks) +INTERFACE_FUNCTION(__sanitizer_purge_allocator) INTERFACE_FUNCTION(__sanitizer_print_memory_profile) INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook) INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc index cf200512de84..5cdfbbb275f7 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -14,7 +14,10 @@ #include "sanitizer_common.h" #include "sanitizer_allocator_interface.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" +#include "sanitizer_procmaps.h" +#include "sanitizer_report_decorator.h" #include "sanitizer_stackdepot.h" #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" @@ -25,12 +28,25 @@ namespace __sanitizer { +#if !SANITIZER_FUCHSIA + bool ReportFile::SupportsColors() { SpinMutexLock l(mu); ReopenIfNecessary(); return SupportsColoredOutput(fd); } +static INLINE bool ReportSupportsColors() { + return report_file.SupportsColors(); +} + +#else // SANITIZER_FUCHSIA + +// Fuchsia's logs always go through post-processing that handles colorization. +static INLINE bool ReportSupportsColors() { return true; } + +#endif // !SANITIZER_FUCHSIA + bool ColorizeReports() { // FIXME: Add proper Windows support to AnsiColorDecorator and re-enable color // printing on Windows. @@ -39,7 +55,7 @@ bool ColorizeReports() { const char *flag = common_flags()->color; return internal_strcmp(flag, "always") == 0 || - (internal_strcmp(flag, "auto") == 0 && report_file.SupportsColors()); + (internal_strcmp(flag, "auto") == 0 && ReportSupportsColors()); } static void (*sandboxing_callback)(); @@ -131,6 +147,127 @@ void BackgroundThread(void *arg) { } #endif +#if !SANITIZER_FUCHSIA && !SANITIZER_GO +void StartReportDeadlySignal() { + // Write the first message using fd=2, just in case. + // It may actually fail to write in case stderr is closed. + CatastrophicErrorWrite(SanitizerToolName, internal_strlen(SanitizerToolName)); + static const char kDeadlySignal[] = ":DEADLYSIGNAL\n"; + CatastrophicErrorWrite(kDeadlySignal, sizeof(kDeadlySignal) - 1); +} + +static void MaybeReportNonExecRegion(uptr pc) { +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD + MemoryMappingLayout proc_maps(/*cache_enabled*/ true); + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if (pc >= segment.start && pc < segment.end && !segment.IsExecutable()) + Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n"); + } +#endif +} + +static void PrintMemoryByte(InternalScopedString *str, const char *before, + u8 byte) { + SanitizerCommonDecorator d; + str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15, + d.Default()); +} + +static void MaybeDumpInstructionBytes(uptr pc) { + if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached())) + return; + InternalScopedString str(1024); + str.append("First 16 instruction bytes at pc: "); + if (IsAccessibleMemoryRange(pc, 16)) { + for (int i = 0; i < 16; ++i) { + PrintMemoryByte(&str, "", ((u8 *)pc)[i]); + } + str.append("\n"); + } else { + str.append("unaccessible\n"); + } + Report("%s", str.data()); +} + +static void MaybeDumpRegisters(void *context) { + if (!common_flags()->dump_registers) return; + SignalContext::DumpAllRegisters(context); +} + +static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + SanitizerCommonDecorator d; + Printf("%s", d.Warning()); + static const char kDescription[] = "stack-overflow"; + Report("ERROR: %s: %s on address %p (pc %p bp %p sp %p T%d)\n", + SanitizerToolName, kDescription, (void *)sig.addr, (void *)sig.pc, + (void *)sig.bp, (void *)sig.sp, tid); + Printf("%s", d.Default()); + InternalScopedBuffer<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + unwind(sig, unwind_context, stack); + stack->Print(); + ReportErrorSummary(kDescription, stack); +} + +static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + SanitizerCommonDecorator d; + Printf("%s", d.Warning()); + const char *description = sig.Describe(); + Report("ERROR: %s: %s on unknown address %p (pc %p bp %p sp %p T%d)\n", + SanitizerToolName, description, (void *)sig.addr, (void *)sig.pc, + (void *)sig.bp, (void *)sig.sp, tid); + Printf("%s", d.Default()); + if (sig.pc < GetPageSizeCached()) + Report("Hint: pc points to the zero page.\n"); + if (sig.is_memory_access) { + const char *access_type = + sig.write_flag == SignalContext::WRITE + ? "WRITE" + : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN"); + Report("The signal is caused by a %s memory access.\n", access_type); + if (sig.addr < GetPageSizeCached()) + Report("Hint: address points to the zero page.\n"); + } + MaybeReportNonExecRegion(sig.pc); + InternalScopedBuffer<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + unwind(sig, unwind_context, stack); + stack->Print(); + MaybeDumpInstructionBytes(sig.pc); + MaybeDumpRegisters(sig.context); + Printf("%s can not provide additional info.\n", SanitizerToolName); + ReportErrorSummary(description, stack); +} + +void ReportDeadlySignal(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + if (sig.IsStackOverflow()) + ReportStackOverflowImpl(sig, tid, unwind, unwind_context); + else + ReportDeadlySignalImpl(sig, tid, unwind, unwind_context); +} + +void HandleDeadlySignal(void *siginfo, void *context, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + StartReportDeadlySignal(); + ScopedErrorReportLock rl; + SignalContext sig(siginfo, context); + ReportDeadlySignal(sig, tid, unwind, unwind_context); + Report("ABORTING\n"); + Die(); +} + +#endif // !SANITIZER_FUCHSIA && !SANITIZER_GO + void WriteToSyslog(const char *msg) { InternalScopedString msg_copy(kErrorMessageBufferSize); msg_copy.append("%s", msg); @@ -161,6 +298,47 @@ void MaybeStartBackgroudThread() { #endif } +static atomic_uintptr_t reporting_thread = {0}; +static StaticSpinMutex CommonSanitizerReportMutex; + +ScopedErrorReportLock::ScopedErrorReportLock() { + uptr current = GetThreadSelf(); + for (;;) { + uptr expected = 0; + if (atomic_compare_exchange_strong(&reporting_thread, &expected, current, + memory_order_relaxed)) { + // We've claimed reporting_thread so proceed. + CommonSanitizerReportMutex.Lock(); + return; + } + + if (expected == current) { + // This is either asynch signal or nested error during error reporting. + // Fail simple to avoid deadlocks in Report(). + + // Can't use Report() here because of potential deadlocks in nested + // signal handlers. + CatastrophicErrorWrite(SanitizerToolName, + internal_strlen(SanitizerToolName)); + static const char msg[] = ": nested bug in the same thread, aborting.\n"; + CatastrophicErrorWrite(msg, sizeof(msg) - 1); + + internal__exit(common_flags()->exitcode); + } + + internal_sched_yield(); + } +} + +ScopedErrorReportLock::~ScopedErrorReportLock() { + CommonSanitizerReportMutex.Unlock(); + atomic_store_relaxed(&reporting_thread, 0); +} + +void ScopedErrorReportLock::CheckLocked() { + CommonSanitizerReportMutex.CheckLocked(); +} + } // namespace __sanitizer SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify, diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc new file mode 100644 index 000000000000..c5be48bceace --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc @@ -0,0 +1,240 @@ +//===-- sanitizer_coverage_fuchsia.cc ------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// Sanitizer Coverage Controller for Trace PC Guard, Fuchsia-specific version. +// +// This Fuchsia-specific implementation uses the same basic scheme and the +// same simple '.sancov' file format as the generic implementation. The +// difference is that we just produce a single blob of output for the whole +// program, not a separate one per DSO. We do not sort the PC table and do +// not prune the zeros, so the resulting file is always as large as it +// would be to report 100% coverage. Implicit tracing information about +// the address ranges of DSOs allows offline tools to split the one big +// blob into separate files that the 'sancov' tool can understand. +// +// Unlike the traditional implementation that uses an atexit hook to write +// out data files at the end, the results on Fuchsia do not go into a file +// per se. The 'coverage_dir' option is ignored. Instead, they are stored +// directly into a shared memory object (a Zircon VMO). At exit, that VMO +// is handed over to a system service that's responsible for getting the +// data out to somewhere that it can be fed into the sancov tool (where and +// how is not our problem). + +#include "sanitizer_platform.h" +#if SANITIZER_FUCHSIA +#include "sanitizer_atomic.h" +#include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" + +#include <zircon/process.h> +#include <zircon/sanitizer.h> +#include <zircon/syscalls.h> + +using namespace __sanitizer; // NOLINT + +namespace __sancov { +namespace { + +// TODO(mcgrathr): Move the constant into a header shared with other impls. +constexpr u64 Magic64 = 0xC0BFFFFFFFFFFF64ULL; +static_assert(SANITIZER_WORDSIZE == 64, "Fuchsia is always LP64"); + +constexpr const char kSancovSinkName[] = "sancov"; + +// Collects trace-pc guard coverage. +// This class relies on zero-initialization. +class TracePcGuardController { + public: + // For each PC location being tracked, there is a u32 reserved in global + // data called the "guard". At startup, we assign each guard slot a + // unique index into the big results array. Later during runtime, the + // first call to TracePcGuard (below) will store the corresponding PC at + // that index in the array. (Each later call with the same guard slot is + // presumed to be from the same PC.) Then it clears the guard slot back + // to zero, which tells the compiler not to bother calling in again. At + // the end of the run, we have a big array where each element is either + // zero or is a tracked PC location that was hit in the trace. + + // This is called from global constructors. Each translation unit has a + // contiguous array of guard slots, and a constructor that calls here + // with the bounds of its array. Those constructors are allowed to call + // here more than once for the same array. Usually all of these + // constructors run in the initial thread, but it's possible that a + // dlopen call on a secondary thread will run constructors that get here. + void InitTracePcGuard(u32 *start, u32 *end) { + if (end > start && *start == 0 && common_flags()->coverage) { + // Complete the setup before filling in any guards with indices. + // This avoids the possibility of code called from Setup reentering + // TracePcGuard. + u32 idx = Setup(end - start); + for (u32 *p = start; p < end; ++p) { + *p = idx++; + } + } + } + + void TracePcGuard(u32 *guard, uptr pc) { + atomic_uint32_t *guard_ptr = reinterpret_cast<atomic_uint32_t *>(guard); + u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed); + if (idx > 0) array_[idx] = pc; + } + + void Dump() { + BlockingMutexLock locked(&setup_lock_); + if (array_) { + CHECK_NE(vmo_, ZX_HANDLE_INVALID); + + // Publish the VMO to the system, where it can be collected and + // analyzed after this process exits. This always consumes the VMO + // handle. Any failure is just logged and not indicated to us. + __sanitizer_publish_data(kSancovSinkName, vmo_); + vmo_ = ZX_HANDLE_INVALID; + + // This will route to __sanitizer_log_write, which will ensure that + // information about shared libraries is written out. This message + // uses the `dumpfile` symbolizer markup element to highlight the + // dump. See the explanation for this in: + // https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md + Printf("SanitizerCoverage: {{{dumpfile:%s:%s}}} with up to %u PCs\n", + kSancovSinkName, vmo_name_, next_index_ - 1); + } + } + + private: + // We map in the largest possible view into the VMO: one word + // for every possible 32-bit index value. This avoids the need + // to change the mapping when increasing the size of the VMO. + // We can always spare the 32G of address space. + static constexpr size_t MappingSize = sizeof(uptr) << 32; + + BlockingMutex setup_lock_; + uptr *array_; + u32 next_index_; + zx_handle_t vmo_; + char vmo_name_[ZX_MAX_NAME_LEN]; + + size_t DataSize() const { return next_index_ * sizeof(uintptr_t); } + + u32 Setup(u32 num_guards) { + BlockingMutexLock locked(&setup_lock_); + DCHECK(common_flags()->coverage); + + if (next_index_ == 0) { + CHECK_EQ(vmo_, ZX_HANDLE_INVALID); + CHECK_EQ(array_, nullptr); + + // The first sample goes at [1] to reserve [0] for the magic number. + next_index_ = 1 + num_guards; + + zx_status_t status = _zx_vmo_create(DataSize(), 0, &vmo_); + CHECK_EQ(status, ZX_OK); + + // Give the VMO a name including our process KOID so it's easy to spot. + internal_snprintf(vmo_name_, sizeof(vmo_name_), "%s.%zu", kSancovSinkName, + internal_getpid()); + _zx_object_set_property(vmo_, ZX_PROP_NAME, vmo_name_, + internal_strlen(vmo_name_)); + + // Map the largest possible view we might need into the VMO. Later + // we might need to increase the VMO's size before we can use larger + // indices, but we'll never move the mapping address so we don't have + // any multi-thread synchronization issues with that. + uintptr_t mapping; + status = + _zx_vmar_map(_zx_vmar_root_self(), 0, vmo_, 0, MappingSize, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &mapping); + CHECK_EQ(status, ZX_OK); + + // Hereafter other threads are free to start storing into + // elements [1, next_index_) of the big array. + array_ = reinterpret_cast<uptr *>(mapping); + + // Store the magic number. + // Hereafter, the VMO serves as the contents of the '.sancov' file. + array_[0] = Magic64; + + return 1; + } else { + // The VMO is already mapped in, but it's not big enough to use the + // new indices. So increase the size to cover the new maximum index. + + CHECK_NE(vmo_, ZX_HANDLE_INVALID); + CHECK_NE(array_, nullptr); + + uint32_t first_index = next_index_; + next_index_ += num_guards; + + zx_status_t status = _zx_vmo_set_size(vmo_, DataSize()); + CHECK_EQ(status, ZX_OK); + + return first_index; + } + } +}; + +static TracePcGuardController pc_guard_controller; + +} // namespace +} // namespace __sancov + +namespace __sanitizer { +void InitializeCoverage(bool enabled, const char *dir) { + CHECK_EQ(enabled, common_flags()->coverage); + CHECK_EQ(dir, common_flags()->coverage_dir); + + static bool coverage_enabled = false; + if (!coverage_enabled) { + coverage_enabled = enabled; + Atexit(__sanitizer_cov_dump); + AddDieCallback(__sanitizer_cov_dump); + } +} +} // namespace __sanitizer + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage( // NOLINT + const uptr *pcs, uptr len) { + UNIMPLEMENTED(); +} + +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *guard) { + if (!*guard) return; + __sancov::pc_guard_controller.TracePcGuard(guard, GET_CALLER_PC() - 1); +} + +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init, + u32 *start, u32 *end) { + if (start == end || *start) return; + __sancov::pc_guard_controller.InitTracePcGuard(start, end); +} + +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() { + __sancov::pc_guard_controller.Dump(); +} +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { + __sanitizer_dump_trace_pc_guard_coverage(); +} +// Default empty implementations (weak). Users should redefine them. +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {} +} // extern "C" + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc index d4749000d80a..f909b74c9752 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc @@ -9,6 +9,7 @@ // Sanitizer Coverage interface list. //===----------------------------------------------------------------------===// INTERFACE_FUNCTION(__sanitizer_cov_dump) +INTERFACE_FUNCTION(__sanitizer_cov_reset) INTERFACE_FUNCTION(__sanitizer_dump_coverage) INTERFACE_FUNCTION(__sanitizer_dump_trace_pc_guard_coverage) INTERFACE_WEAK_FUNCTION(__sancov_default_options) @@ -17,6 +18,10 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp1) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp2) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp4) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp8) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp1) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp2) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp4) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp8) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div4) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div8) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep) @@ -24,3 +29,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_pcs_init) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc index 24433356c63d..3c5f29b2899b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc @@ -8,10 +8,14 @@ //===----------------------------------------------------------------------===// // Sanitizer Coverage Controller for Trace PC Guard. +#include "sanitizer_platform.h" + +#if !SANITIZER_FUCHSIA #include "sancov_flags.h" #include "sanitizer_allocator_internal.h" #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_symbolizer.h" using namespace __sanitizer; @@ -124,11 +128,17 @@ class TracePcGuardController { } void TracePcGuard(u32* guard, uptr pc) { - atomic_uint32_t* guard_ptr = reinterpret_cast<atomic_uint32_t*>(guard); - u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed); + u32 idx = *guard; if (!idx) return; // we start indices from 1. - pc_vector[idx - 1] = pc; + atomic_uintptr_t* pc_ptr = + reinterpret_cast<atomic_uintptr_t*>(&pc_vector[idx - 1]); + if (atomic_load(pc_ptr, memory_order_relaxed) == 0) + atomic_store(pc_ptr, pc, memory_order_relaxed); + } + + void Reset() { + internal_memset(&pc_vector[0], 0, sizeof(pc_vector[0]) * pc_vector.size()); } void Dump() { @@ -180,15 +190,31 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() { SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { __sanitizer_dump_trace_pc_guard_coverage(); } +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_reset() { + __sancov::pc_guard_controller.Reset(); +} // Default empty implementations (weak). Users should redefine them. SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {} } // extern "C" +// Weak definition for code instrumented with -fsanitize-coverage=stack-depth +// and later linked with code containing a strong definition. +// E.g., -fsanitize=fuzzer-no-link +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE uptr __sancov_lowest_stack; + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h index 7872b89c227c..42cc290502bf 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h @@ -24,8 +24,10 @@ #if SANITIZER_FREEBSD || SANITIZER_MAC # define __errno_location __error -#elif SANITIZER_ANDROID +#elif SANITIZER_ANDROID || SANITIZER_NETBSD # define __errno_location __errno +#elif SANITIZER_SOLARIS +# define __errno_location ___errno #elif SANITIZER_WINDOWS # define __errno_location _errno #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc new file mode 100644 index 000000000000..cde54bfde804 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc @@ -0,0 +1,177 @@ +//===-- sanitizer_file.cc ------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is shared between AddressSanitizer and ThreadSanitizer +// run-time libraries. It defines filesystem-related interfaces. This +// is separate from sanitizer_common.cc so that it's simpler to disable +// all the filesystem support code for a port that doesn't use it. +// +//===---------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if !SANITIZER_FUCHSIA + +#include "sanitizer_common.h" +#include "sanitizer_file.h" + +namespace __sanitizer { + +void CatastrophicErrorWrite(const char *buffer, uptr length) { + WriteToFile(kStderrFd, buffer, length); +} + +StaticSpinMutex report_file_mu; +ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0}; + +void RawWrite(const char *buffer) { + report_file.Write(buffer, internal_strlen(buffer)); +} + +void ReportFile::ReopenIfNecessary() { + mu->CheckLocked(); + if (fd == kStdoutFd || fd == kStderrFd) return; + + uptr pid = internal_getpid(); + // If in tracer, use the parent's file. + if (pid == stoptheworld_tracer_pid) + pid = stoptheworld_tracer_ppid; + if (fd != kInvalidFd) { + // If the report file is already opened by the current process, + // do nothing. Otherwise the report file was opened by the parent + // process, close it now. + if (fd_pid == pid) + return; + else + CloseFile(fd); + } + + const char *exe_name = GetProcessName(); + if (common_flags()->log_exe_name && exe_name) { + internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix, + exe_name, pid); + } else { + internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid); + } + fd = OpenFile(full_path, WrOnly); + if (fd == kInvalidFd) { + const char *ErrorMsgPrefix = "ERROR: Can't open file: "; + WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix)); + WriteToFile(kStderrFd, full_path, internal_strlen(full_path)); + Die(); + } + fd_pid = pid; +} + +void ReportFile::SetReportPath(const char *path) { + if (!path) + return; + uptr len = internal_strlen(path); + if (len > sizeof(path_prefix) - 100) { + Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n", + path[0], path[1], path[2], path[3], + path[4], path[5], path[6], path[7]); + Die(); + } + + SpinMutexLock l(mu); + if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd) + CloseFile(fd); + fd = kInvalidFd; + if (internal_strcmp(path, "stdout") == 0) { + fd = kStdoutFd; + } else if (internal_strcmp(path, "stderr") == 0) { + fd = kStderrFd; + } else { + internal_snprintf(path_prefix, kMaxPathLength, "%s", path); + } +} + +bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr *read_len, uptr max_len, error_t *errno_p) { + uptr PageSize = GetPageSizeCached(); + uptr kMinFileLen = PageSize; + *buff = nullptr; + *buff_size = 0; + *read_len = 0; + // The files we usually open are not seekable, so try different buffer sizes. + for (uptr size = kMinFileLen; size <= max_len; size *= 2) { + fd_t fd = OpenFile(file_name, RdOnly, errno_p); + if (fd == kInvalidFd) return false; + UnmapOrDie(*buff, *buff_size); + *buff = (char*)MmapOrDie(size, __func__); + *buff_size = size; + *read_len = 0; + // Read up to one page at a time. + bool reached_eof = false; + while (*read_len + PageSize <= size) { + uptr just_read; + if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) { + UnmapOrDie(*buff, *buff_size); + return false; + } + if (just_read == 0) { + reached_eof = true; + break; + } + *read_len += just_read; + } + CloseFile(fd); + if (reached_eof) // We've read the whole file. + break; + } + return true; +} + +static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':'; + +char *FindPathToBinary(const char *name) { + if (FileExists(name)) { + return internal_strdup(name); + } + + const char *path = GetEnv("PATH"); + if (!path) + return nullptr; + uptr name_len = internal_strlen(name); + InternalScopedBuffer<char> buffer(kMaxPathLength); + const char *beg = path; + while (true) { + const char *end = internal_strchrnul(beg, kPathSeparator); + uptr prefix_len = end - beg; + if (prefix_len + name_len + 2 <= kMaxPathLength) { + internal_memcpy(buffer.data(), beg, prefix_len); + buffer[prefix_len] = '/'; + internal_memcpy(&buffer[prefix_len + 1], name, name_len); + buffer[prefix_len + 1 + name_len] = '\0'; + if (FileExists(buffer.data())) + return internal_strdup(buffer.data()); + } + if (*end == '\0') break; + beg = end + 1; + } + return nullptr; +} + +} // namespace __sanitizer + +using namespace __sanitizer; // NOLINT + +extern "C" { +void __sanitizer_set_report_path(const char *path) { + report_file.SetReportPath(path); +} + +void __sanitizer_set_report_fd(void *fd) { + report_file.fd = (fd_t)reinterpret_cast<uptr>(fd); + report_file.fd_pid = internal_getpid(); +} +} // extern "C" + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h new file mode 100644 index 000000000000..9a12ab7342c7 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h @@ -0,0 +1,110 @@ +//===-- sanitizer_file.h ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is shared between run-time libraries of sanitizers. +// It declares filesystem-related interfaces. This is separate from +// sanitizer_common.h so that it's simpler to disable all the filesystem +// support code for a port that doesn't use it. +// +//===---------------------------------------------------------------------===// +#ifndef SANITIZER_FILE_H +#define SANITIZER_FILE_H + +#include "sanitizer_interface_internal.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_libc.h" +#include "sanitizer_mutex.h" + +namespace __sanitizer { + +struct ReportFile { + void Write(const char *buffer, uptr length); + bool SupportsColors(); + void SetReportPath(const char *path); + + // Don't use fields directly. They are only declared public to allow + // aggregate initialization. + + // Protects fields below. + StaticSpinMutex *mu; + // Opened file descriptor. Defaults to stderr. It may be equal to + // kInvalidFd, in which case new file will be opened when necessary. + fd_t fd; + // Path prefix of report file, set via __sanitizer_set_report_path. + char path_prefix[kMaxPathLength]; + // Full path to report, obtained as <path_prefix>.PID + char full_path[kMaxPathLength]; + // PID of the process that opened fd. If a fork() occurs, + // the PID of child will be different from fd_pid. + uptr fd_pid; + + private: + void ReopenIfNecessary(); +}; +extern ReportFile report_file; + +enum FileAccessMode { + RdOnly, + WrOnly, + RdWr +}; + +// Returns kInvalidFd on error. +fd_t OpenFile(const char *filename, FileAccessMode mode, + error_t *errno_p = nullptr); +void CloseFile(fd_t); + +// Return true on success, false on error. +bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, + uptr *bytes_read = nullptr, error_t *error_p = nullptr); +bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, + uptr *bytes_written = nullptr, error_t *error_p = nullptr); + +bool RenameFile(const char *oldpath, const char *newpath, + error_t *error_p = nullptr); + +// Scoped file handle closer. +struct FileCloser { + explicit FileCloser(fd_t fd) : fd(fd) {} + ~FileCloser() { CloseFile(fd); } + fd_t fd; +}; + +bool SupportsColoredOutput(fd_t fd); + +// OS +const char *GetPwd(); +bool FileExists(const char *filename); +char *FindPathToBinary(const char *name); +bool IsPathSeparator(const char c); +bool IsAbsolutePath(const char *path); +// Starts a subprocess and returs its pid. +// If *_fd parameters are not kInvalidFd their corresponding input/output +// streams will be redirect to the file. The files will always be closed +// in parent process even in case of an error. +// The child process will close all fds after STDERR_FILENO +// before passing control to a program. +pid_t StartSubprocess(const char *filename, const char *const argv[], + fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd, + fd_t stderr_fd = kInvalidFd); +// Checks if specified process is still running +bool IsProcessRunning(pid_t pid); +// Waits for the process to finish and returns its exit code. +// Returns -1 in case of an error. +int WaitForProcess(pid_t pid); + +// Maps given file to virtual memory, and returns pointer to it +// (or NULL if mapping fails). Stores the size of mmaped region +// in '*buff_size'. +void *MapFileToMemory(const char *file_name, uptr *buff_size); +void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset); + +} // namespace __sanitizer + +#endif // SANITIZER_FILE_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h index 4988fbb7a86d..f649f5bffea7 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h @@ -75,7 +75,7 @@ inline bool FlagHandler<HandleSignalMode>::Parse(const char *value) { template <> inline bool FlagHandler<const char *>::Parse(const char *value) { - *t_ = internal_strdup(value); + *t_ = value; return true; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc index 9f71861efcdb..445e25f9df70 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc @@ -62,6 +62,9 @@ COMMON_FLAG( COMMON_FLAG( int, verbosity, 0, "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).") +COMMON_FLAG(bool, strip_env, 1, + "Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to " + "avoid passing it to children. Default is true.") COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.") COMMON_FLAG( bool, leak_check_at_exit, true, @@ -129,11 +132,11 @@ COMMON_FLAG(uptr, soft_rss_limit_mb, 0, " This limit does not affect memory allocations other than" " malloc/new.") COMMON_FLAG(bool, heap_profile, false, "Experimental heap profiler, asan-only") -COMMON_FLAG(s32, allocator_release_to_os_interval_ms, kReleaseToOSIntervalNever, - "Experimental. Only affects a 64-bit allocator. If set, tries to " - "release unused memory to the OS, but not more often than this " - "interval (in milliseconds). Negative values mean do not attempt " - "to release memory to the OS.\n") +COMMON_FLAG(s32, allocator_release_to_os_interval_ms, 5000, + "Only affects a 64-bit allocator. If set, tries to release unused " + "memory to the OS, but not more often than this interval (in " + "milliseconds). Negative values mean do not attempt to release " + "memory to the OS.\n") COMMON_FLAG(bool, can_use_proc_maps_statm, true, "If false, do not attempt to read /proc/maps/statm." " Mostly useful for testing sanitizers.") @@ -229,3 +232,8 @@ COMMON_FLAG(bool, print_cmdline, false, "Print command line on crash " "(asan only).") COMMON_FLAG(bool, html_cov_report, false, "Generate html coverage report.") COMMON_FLAG(const char *, sancov_path, "sancov", "Sancov tool location.") +COMMON_FLAG(bool, dump_instruction_bytes, false, + "If true, dump 16 bytes starting at the instruction that caused SEGV") +COMMON_FLAG(bool, dump_registers, true, + "If true, dump values of CPU registers when SEGV happens. Only " + "available on OS X for now.") diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc new file mode 100644 index 000000000000..936ec794b8e8 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc @@ -0,0 +1,539 @@ +//===-- sanitizer_fuchsia.cc ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is shared between AddressSanitizer and other sanitizer +// run-time libraries and implements Fuchsia-specific functions from +// sanitizer_common.h. +//===---------------------------------------------------------------------===// + +#include "sanitizer_fuchsia.h" +#if SANITIZER_FUCHSIA + +#include "sanitizer_common.h" +#include "sanitizer_libc.h" +#include "sanitizer_mutex.h" +#include "sanitizer_stacktrace.h" + +#include <limits.h> +#include <pthread.h> +#include <stdlib.h> +#include <unistd.h> +#include <unwind.h> +#include <zircon/errors.h> +#include <zircon/process.h> +#include <zircon/syscalls.h> + +namespace __sanitizer { + +void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); } + +uptr internal_sched_yield() { + zx_status_t status = _zx_nanosleep(0); + CHECK_EQ(status, ZX_OK); + return 0; // Why doesn't this return void? +} + +static void internal_nanosleep(zx_time_t ns) { + zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns)); + CHECK_EQ(status, ZX_OK); +} + +unsigned int internal_sleep(unsigned int seconds) { + internal_nanosleep(ZX_SEC(seconds)); + return 0; +} + +u64 NanoTime() { return _zx_time_get(ZX_CLOCK_UTC); } + +u64 MonotonicNanoTime() { return _zx_time_get(ZX_CLOCK_MONOTONIC); } + +uptr internal_getpid() { + zx_info_handle_basic_t info; + zx_status_t status = + _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &info, + sizeof(info), NULL, NULL); + CHECK_EQ(status, ZX_OK); + uptr pid = static_cast<uptr>(info.koid); + CHECK_EQ(pid, info.koid); + return pid; +} + +uptr GetThreadSelf() { return reinterpret_cast<uptr>(thrd_current()); } + +uptr GetTid() { return GetThreadSelf(); } + +void Abort() { abort(); } + +int Atexit(void (*function)(void)) { return atexit(function); } + +void SleepForSeconds(int seconds) { internal_sleep(seconds); } + +void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); } + +void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) { + pthread_attr_t attr; + CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); + void *base; + size_t size; + CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0); + CHECK_EQ(pthread_attr_destroy(&attr), 0); + + *stack_bottom = reinterpret_cast<uptr>(base); + *stack_top = *stack_bottom + size; +} + +void MaybeReexec() {} +void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {} +void DisableCoreDumperIfNecessary() {} +void InstallDeadlySignalHandlers(SignalHandlerType handler) {} +void StartReportDeadlySignal() {} +void ReportDeadlySignal(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) {} +void SetAlternateSignalStack() {} +void UnsetAlternateSignalStack() {} +void InitTlsSize() {} + +void PrintModuleMap() {} + +bool SignalContext::IsStackOverflow() const { return false; } +void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); } +const char *SignalContext::Describe() const { UNIMPLEMENTED(); } + +struct UnwindTraceArg { + BufferedStackTrace *stack; + u32 max_depth; +}; + +_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { + UnwindTraceArg *arg = static_cast<UnwindTraceArg *>(param); + CHECK_LT(arg->stack->size, arg->max_depth); + uptr pc = _Unwind_GetIP(ctx); + if (pc < PAGE_SIZE) return _URC_NORMAL_STOP; + arg->stack->trace_buffer[arg->stack->size++] = pc; + return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP + : _URC_NO_REASON); +} + +void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { + CHECK_GE(max_depth, 2); + size = 0; + UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)}; + _Unwind_Backtrace(Unwind_Trace, &arg); + CHECK_GT(size, 0); + // We need to pop a few frames so that pc is on top. + uptr to_pop = LocatePcInTrace(pc); + // trace_buffer[0] belongs to the current function so we always pop it, + // unless there is only 1 frame in the stack trace (1 frame is always better + // than 0!). + PopStackFrames(Min(to_pop, static_cast<uptr>(1))); + trace_buffer[0] = pc; +} + +void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, + u32 max_depth) { + CHECK_NE(context, nullptr); + UNREACHABLE("signal context doesn't exist"); +} + +enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 }; + +BlockingMutex::BlockingMutex() { + // NOTE! It's important that this use internal_memset, because plain + // memset might be intercepted (e.g., actually be __asan_memset). + // Defining this so the compiler initializes each field, e.g.: + // BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {} + // might result in the compiler generating a call to memset, which would + // have the same problem. + internal_memset(this, 0, sizeof(*this)); +} + +void BlockingMutex::Lock() { + CHECK_EQ(owner_, 0); + atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); + if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked) + return; + while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) { + zx_status_t status = _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m), + MtxSleeping, ZX_TIME_INFINITE); + if (status != ZX_ERR_BAD_STATE) // Normal race. + CHECK_EQ(status, ZX_OK); + } +} + +void BlockingMutex::Unlock() { + atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); + u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release); + CHECK_NE(v, MtxUnlocked); + if (v == MtxSleeping) { + zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(m), 1); + CHECK_EQ(status, ZX_OK); + } +} + +void BlockingMutex::CheckLocked() { + atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); + CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed)); +} + +uptr GetPageSize() { return PAGE_SIZE; } + +uptr GetMmapGranularity() { return PAGE_SIZE; } + +sanitizer_shadow_bounds_t ShadowBounds; + +uptr GetMaxUserVirtualAddress() { + ShadowBounds = __sanitizer_shadow_bounds(); + return ShadowBounds.memory_limit - 1; +} + +uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); } + +static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, + bool raw_report, bool die_for_nomem) { + size = RoundUpTo(size, PAGE_SIZE); + + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(size, 0, &vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) + ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status, + raw_report); + return nullptr; + } + _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type, + internal_strlen(mem_type)); + + // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? + uintptr_t addr; + status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + _zx_handle_close(vmo); + + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) + ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status, + raw_report); + return nullptr; + } + + IncreaseTotalMmap(size); + + return reinterpret_cast<void *>(addr); +} + +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { + return DoAnonymousMmapOrDie(size, mem_type, raw_report, true); +} + +void *MmapNoReserveOrDie(uptr size, const char *mem_type) { + return MmapOrDie(size, mem_type); +} + +void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { + return DoAnonymousMmapOrDie(size, mem_type, false, false); +} + +uptr ReservedAddressRange::Init(uptr init_size, const char *name, + uptr fixed_addr) { + init_size = RoundUpTo(init_size, PAGE_SIZE); + DCHECK_EQ(os_handle_, ZX_HANDLE_INVALID); + uintptr_t base; + zx_handle_t vmar; + zx_status_t status = + _zx_vmar_allocate(_zx_vmar_root_self(), 0, init_size, + ZX_VM_FLAG_CAN_MAP_READ | ZX_VM_FLAG_CAN_MAP_WRITE | + ZX_VM_FLAG_CAN_MAP_SPECIFIC, + &vmar, &base); + if (status != ZX_OK) + ReportMmapFailureAndDie(init_size, name, "zx_vmar_allocate", status); + base_ = reinterpret_cast<void *>(base); + size_ = init_size; + name_ = name; + os_handle_ = vmar; + + return reinterpret_cast<uptr>(base_); +} + +static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size, + void *base, const char *name, bool die_for_nomem) { + uptr offset = fixed_addr - reinterpret_cast<uptr>(base); + map_size = RoundUpTo(map_size, PAGE_SIZE); + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(map_size, 0, &vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) + ReportMmapFailureAndDie(map_size, name, "zx_vmo_create", status); + return 0; + } + _zx_object_set_property(vmo, ZX_PROP_NAME, name, sizeof(name) - 1); + DCHECK_GE(base + size_, map_size + offset); + uintptr_t addr; + + status = _zx_vmar_map( + vmar, offset, vmo, 0, map_size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | ZX_VM_FLAG_SPECIFIC, + &addr); + _zx_handle_close(vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) { + ReportMmapFailureAndDie(map_size, name, "zx_vmar_map", status); + } + return 0; + } + IncreaseTotalMmap(map_size); + return addr; +} + +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size) { + return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, + name_, false); +} + +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size) { + return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, + name_, true); +} + +void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) { + if (!addr || !size) return; + size = RoundUpTo(size, PAGE_SIZE); + + zx_status_t status = + _zx_vmar_unmap(target_vmar, reinterpret_cast<uintptr_t>(addr), size); + if (status != ZX_OK) { + Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n", + SanitizerToolName, size, size, addr); + CHECK("unable to unmap" && 0); + } + + DecreaseTotalMmap(size); +} + +void ReservedAddressRange::Unmap(uptr fixed_addr, uptr size) { + uptr offset = fixed_addr - reinterpret_cast<uptr>(base_); + uptr addr = reinterpret_cast<uptr>(base_) + offset; + void *addr_as_void = reinterpret_cast<void *>(addr); + uptr base_as_uptr = reinterpret_cast<uptr>(base_); + // Only unmap at the beginning or end of the range. + CHECK((addr_as_void == base_) || (addr + size == base_as_uptr + size_)); + CHECK_LE(size, size_); + UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, + static_cast<zx_handle_t>(os_handle_)); + if (addr_as_void == base_) { + base_ = reinterpret_cast<void *>(addr + size); + } + size_ = size_ - size; +} + +// This should never be called. +void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { + UNIMPLEMENTED(); +} + +void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, + const char *mem_type) { + CHECK_GE(size, PAGE_SIZE); + CHECK(IsPowerOfTwo(size)); + CHECK(IsPowerOfTwo(alignment)); + + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(size, 0, &vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY) + ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status, false); + return nullptr; + } + _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type, + internal_strlen(mem_type)); + + // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? + + // Map a larger size to get a chunk of address space big enough that + // it surely contains an aligned region of the requested size. Then + // overwrite the aligned middle portion with a mapping from the + // beginning of the VMO, and unmap the excess before and after. + size_t map_size = size + alignment; + uintptr_t addr; + status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + if (status == ZX_OK) { + uintptr_t map_addr = addr; + uintptr_t map_end = map_addr + map_size; + addr = RoundUpTo(map_addr, alignment); + uintptr_t end = addr + size; + if (addr != map_addr) { + zx_info_vmar_t info; + status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info, + sizeof(info), NULL, NULL); + if (status == ZX_OK) { + uintptr_t new_addr; + status = + _zx_vmar_map(_zx_vmar_root_self(), addr - info.base, vmo, 0, size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | + ZX_VM_FLAG_SPECIFIC_OVERWRITE, + &new_addr); + if (status == ZX_OK) CHECK_EQ(new_addr, addr); + } + } + if (status == ZX_OK && addr != map_addr) + status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr); + if (status == ZX_OK && end != map_end) + status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end); + } + _zx_handle_close(vmo); + + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY) + ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status, false); + return nullptr; + } + + IncreaseTotalMmap(size); + + return reinterpret_cast<void *>(addr); +} + +void UnmapOrDie(void *addr, uptr size) { + UnmapOrDieVmar(addr, size, _zx_vmar_root_self()); +} + +// This is used on the shadow mapping, which cannot be changed. +// Zircon doesn't have anything like MADV_DONTNEED. +void ReleaseMemoryPagesToOS(uptr beg, uptr end) {} + +void DumpProcessMap() { + // TODO(mcgrathr): write it + return; +} + +bool IsAccessibleMemoryRange(uptr beg, uptr size) { + // TODO(mcgrathr): Figure out a better way. + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(size, 0, &vmo); + if (status == ZX_OK) { + while (size > 0) { + size_t wrote; + status = _zx_vmo_write(vmo, reinterpret_cast<const void *>(beg), 0, size, + &wrote); + if (status != ZX_OK) break; + CHECK_GT(wrote, 0); + CHECK_LE(wrote, size); + beg += wrote; + size -= wrote; + } + _zx_handle_close(vmo); + } + return status == ZX_OK; +} + +// FIXME implement on this platform. +void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {} + +bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr *read_len, uptr max_len, error_t *errno_p) { + zx_handle_t vmo; + zx_status_t status = __sanitizer_get_configuration(file_name, &vmo); + if (status == ZX_OK) { + uint64_t vmo_size; + status = _zx_vmo_get_size(vmo, &vmo_size); + if (status == ZX_OK) { + if (vmo_size < max_len) max_len = vmo_size; + size_t map_size = RoundUpTo(max_len, PAGE_SIZE); + uintptr_t addr; + status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size, + ZX_VM_FLAG_PERM_READ, &addr); + if (status == ZX_OK) { + *buff = reinterpret_cast<char *>(addr); + *buff_size = map_size; + *read_len = max_len; + } + } + _zx_handle_close(vmo); + } + if (status != ZX_OK && errno_p) *errno_p = status; + return status == ZX_OK; +} + +void RawWrite(const char *buffer) { + __sanitizer_log_write(buffer, internal_strlen(buffer)); +} + +void CatastrophicErrorWrite(const char *buffer, uptr length) { + __sanitizer_log_write(buffer, length); +} + +char **StoredArgv; +char **StoredEnviron; + +char **GetArgv() { return StoredArgv; } + +const char *GetEnv(const char *name) { + if (StoredEnviron) { + uptr NameLen = internal_strlen(name); + for (char **Env = StoredEnviron; *Env != 0; Env++) { + if (internal_strncmp(*Env, name, NameLen) == 0 && (*Env)[NameLen] == '=') + return (*Env) + NameLen + 1; + } + } + return nullptr; +} + +uptr ReadBinaryName(/*out*/ char *buf, uptr buf_len) { + const char *argv0 = StoredArgv[0]; + if (!argv0) argv0 = "<UNKNOWN>"; + internal_strncpy(buf, argv0, buf_len); + return internal_strlen(buf); +} + +uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) { + return ReadBinaryName(buf, buf_len); +} + +uptr MainThreadStackBase, MainThreadStackSize; + +bool GetRandom(void *buffer, uptr length, bool blocking) { + CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN); + size_t size; + CHECK_EQ(_zx_cprng_draw(buffer, length, &size), ZX_OK); + CHECK_EQ(size, length); + return true; +} + +u32 GetNumberOfCPUs() { + return zx_system_get_num_cpus(); +} + +uptr GetRSS() { UNIMPLEMENTED(); } + +} // namespace __sanitizer + +using namespace __sanitizer; // NOLINT + +extern "C" { +void __sanitizer_startup_hook(int argc, char **argv, char **envp, + void *stack_base, size_t stack_size) { + __sanitizer::StoredArgv = argv; + __sanitizer::StoredEnviron = envp; + __sanitizer::MainThreadStackBase = reinterpret_cast<uintptr_t>(stack_base); + __sanitizer::MainThreadStackSize = stack_size; +} + +void __sanitizer_set_report_path(const char *path) { + // Handle the initialization code in each sanitizer, but no other calls. + // This setting is never consulted on Fuchsia. + DCHECK_EQ(path, common_flags()->log_path); +} + +void __sanitizer_set_report_fd(void *fd) { + UNREACHABLE("not available on Fuchsia"); +} +} // extern "C" + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h new file mode 100644 index 000000000000..18821b4fd9f1 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h @@ -0,0 +1,31 @@ +//===-- sanitizer_fuchsia.h ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// Fuchsia-specific sanitizer support. +// +//===---------------------------------------------------------------------===// +#ifndef SANITIZER_FUCHSIA_H +#define SANITIZER_FUCHSIA_H + +#include "sanitizer_platform.h" +#if SANITIZER_FUCHSIA + +#include "sanitizer_common.h" + +#include <zircon/sanitizer.h> + +namespace __sanitizer { + +extern uptr MainThreadStackBase, MainThreadStackSize; +extern sanitizer_shadow_bounds_t ShadowBounds; + +} // namespace __sanitizer + +#endif // SANITIZER_FUCHSIA +#endif // SANITIZER_FUCHSIA_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h new file mode 100644 index 000000000000..934e311a7851 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h @@ -0,0 +1,47 @@ +//===-- sanitizer_getauxval.h -----------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Common getauxval() guards and definitions. +// getauxval() is not defined until glibc version 2.16, or until API level 21 +// for Android. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_GETAUXVAL_H +#define SANITIZER_GETAUXVAL_H + +#include "sanitizer_platform.h" + +#if SANITIZER_LINUX + +# include <features.h> + +# ifndef __GLIBC_PREREQ +# define __GLIBC_PREREQ(x, y) 0 +# endif + +# if __GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) +# define SANITIZER_USE_GETAUXVAL 1 +# else +# define SANITIZER_USE_GETAUXVAL 0 +# endif + +# if SANITIZER_USE_GETAUXVAL +# include <sys/auxv.h> +# else +// The weak getauxval definition allows to check for the function at runtime. +// This is useful for Android, when compiled at a lower API level yet running +// on a more recent platform that offers the function. +extern "C" SANITIZER_WEAK_ATTRIBUTE +unsigned long getauxval(unsigned long type); // NOLINT +# endif + +#endif // SANITIZER_LINUX + +#endif // SANITIZER_GETAUXVAL_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h index b28d8f08e7a3..942f0b1cb586 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h @@ -46,7 +46,6 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_report_error_summary(const char *error_summary); - SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init(); SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump(); SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage( const __sanitizer::uptr *pcs, const __sanitizer::uptr len); @@ -81,6 +80,14 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp8(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp1(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp2(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp4(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp8(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_switch(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_div4(); @@ -95,6 +102,10 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*, __sanitizer::u32*); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_8bit_counters_init(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_pcs_init(); } // extern "C" #endif // SANITIZER_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index f35b095ee94e..dc480e75f98d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -35,6 +35,14 @@ # define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) #endif +// TLS is handled differently on different platforms +#if SANITIZER_LINUX || SANITIZER_NETBSD +# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \ + __attribute__((tls_model("initial-exec"))) thread_local +#else +# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE +#endif + //--------------------------- WEAK FUNCTIONS ---------------------------------// // When working with weak functions, to simplify the code and make it more // portable, when possible define a default implementation using this macro: @@ -56,11 +64,13 @@ // SANITIZER_SUPPORTS_WEAK_HOOKS means that we support real weak functions that // will evaluate to a null pointer when not defined. -#if (SANITIZER_LINUX || SANITIZER_MAC) && !SANITIZER_GO +#ifndef SANITIZER_SUPPORTS_WEAK_HOOKS +#if (SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_SOLARIS) && !SANITIZER_GO # define SANITIZER_SUPPORTS_WEAK_HOOKS 1 #else # define SANITIZER_SUPPORTS_WEAK_HOOKS 0 #endif +#endif // SANITIZER_SUPPORTS_WEAK_HOOKS // For some weak hooks that will be called very often and we want to avoid the // overhead of executing the default implementation when it is not necessary, // we can use the flag SANITIZER_SUPPORTS_WEAK_HOOKS to only define the default @@ -81,6 +91,10 @@ // FIXME: do we have anything like this on Mac? #if SANITIZER_LINUX && !SANITIZER_ANDROID && !defined(PIC) # define SANITIZER_CAN_USE_PREINIT_ARRAY 1 +// Before Solaris 11.4, .preinit_array is fully supported only with GNU ld. +// FIXME: Check for those conditions. +#elif SANITIZER_SOLARIS && !defined(PIC) +# define SANITIZER_CAN_USE_PREINIT_ARRAY 1 #else # define SANITIZER_CAN_USE_PREINIT_ARRAY 0 #endif @@ -127,14 +141,14 @@ typedef unsigned error_t; typedef int fd_t; typedef int error_t; #endif +#if SANITIZER_SOLARIS && !defined(_LP64) +typedef long pid_t; +#else typedef int pid_t; +#endif -// WARNING: OFF_T may be different from OS type off_t, depending on the value of -// _FILE_OFFSET_BITS. This definition of OFF_T matches the ABI of system calls -// like pread and mmap, as opposed to pread64 and mmap64. -// FreeBSD, Mac and Linux/x86-64 are special. -#if SANITIZER_FREEBSD || SANITIZER_MAC || \ - (SANITIZER_LINUX && defined(__x86_64__)) +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC || \ + (SANITIZER_LINUX && defined(__x86_64__)) typedef u64 OFF_T; #else typedef uptr OFF_T; @@ -257,8 +271,8 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, #define CHECK_IMPL(c1, op, c2) \ do { \ - __sanitizer::u64 v1 = (u64)(c1); \ - __sanitizer::u64 v2 = (u64)(c2); \ + __sanitizer::u64 v1 = (__sanitizer::u64)(c1); \ + __sanitizer::u64 v2 = (__sanitizer::u64)(c2); \ if (UNLIKELY(!(v1 op v2))) \ __sanitizer::CheckFailed(__FILE__, __LINE__, \ "(" #c1 ") " #op " (" #c2 ")", v1, v2); \ @@ -384,6 +398,7 @@ namespace __dfsan { using namespace __sanitizer; } // NOLINT namespace __esan { using namespace __sanitizer; } // NOLINT namespace __lsan { using namespace __sanitizer; } // NOLINT namespace __msan { using namespace __sanitizer; } // NOLINT +namespace __hwasan { using namespace __sanitizer; } // NOLINT namespace __tsan { using namespace __sanitizer; } // NOLINT namespace __scudo { using namespace __sanitizer; } // NOLINT namespace __ubsan { using namespace __sanitizer; } // NOLINT diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc index aa4fa88efffa..0df055edae97 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc @@ -9,7 +9,7 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD #include "sanitizer_libignore.h" #include "sanitizer_flags.h" @@ -125,4 +125,5 @@ void LibIgnore::OnLibraryUnloaded() { } // namespace __sanitizer -#endif // #if SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || + // SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index 8c3c1e5d6a5d..6c83e8db42a5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -14,10 +14,12 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_flags.h" +#include "sanitizer_getauxval.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" #include "sanitizer_linux.h" @@ -27,10 +29,14 @@ #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX #include <asm/param.h> #endif +#if SANITIZER_NETBSD +#include <lwp.h> +#endif + // For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat' // format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To // access stat from asm/stat.h, without conflicting with definition in @@ -50,7 +56,9 @@ #include <pthread.h> #include <sched.h> #include <sys/mman.h> +#if !SANITIZER_SOLARIS #include <sys/ptrace.h> +#endif #include <sys/resource.h> #include <sys/stat.h> #include <sys/syscall.h> @@ -79,22 +87,24 @@ extern "C" { extern char **environ; // provided by crt1 #endif // SANITIZER_FREEBSD -#if !SANITIZER_ANDROID -#include <sys/signal.h> -#endif +#if SANITIZER_NETBSD +#include <limits.h> // For NAME_MAX +#include <sys/sysctl.h> +extern char **environ; // provided by crt1 +#include <sys/exec.h> +extern struct ps_strings *__ps_strings; +#endif // SANITIZER_NETBSD -#ifndef __GLIBC_PREREQ -#define __GLIBC_PREREQ(x, y) 0 -#endif +#if SANITIZER_SOLARIS +#include <stdlib.h> +#include <thread.h> -#if SANITIZER_LINUX && __GLIBC_PREREQ(2, 16) -# define SANITIZER_USE_GETAUXVAL 1 -#else -# define SANITIZER_USE_GETAUXVAL 0 +extern char **_environ; +#define environ _environ #endif -#if SANITIZER_USE_GETAUXVAL -#include <sys/auxv.h> +#if !SANITIZER_ANDROID +#include <sys/signal.h> #endif #if SANITIZER_LINUX @@ -125,21 +135,36 @@ extern void internal_sigreturn(); } #endif +#if SANITIZER_LINUX && defined(__NR_getrandom) +# if !defined(GRND_NONBLOCK) +# define GRND_NONBLOCK 1 +# endif +# define SANITIZER_USE_GETRANDOM 1 +#else +# define SANITIZER_USE_GETRANDOM 0 +#endif // SANITIZER_LINUX && defined(__NR_getrandom) + namespace __sanitizer { #if SANITIZER_LINUX && defined(__x86_64__) #include "sanitizer_syscall_linux_x86_64.inc" #elif SANITIZER_LINUX && defined(__aarch64__) #include "sanitizer_syscall_linux_aarch64.inc" +#elif SANITIZER_LINUX && defined(__arm__) +#include "sanitizer_syscall_linux_arm.inc" #else #include "sanitizer_syscall_generic.inc" #endif // --------------- sanitizer_libc.h +#if !SANITIZER_SOLARIS #if !SANITIZER_S390 uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, OFF_T offset) { -#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +#if SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(mmap), addr, length, prot, flags, fd, + (long)0, offset); +#elif SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd, offset); #else @@ -152,11 +177,11 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, #endif // !SANITIZER_S390 uptr internal_munmap(void *addr, uptr length) { - return internal_syscall(SYSCALL(munmap), (uptr)addr, length); + return internal_syscall_ptr(SYSCALL(munmap), (uptr)addr, length); } int internal_mprotect(void *addr, uptr length, int prot) { - return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot); + return internal_syscall_ptr(SYSCALL(mprotect), (uptr)addr, length, prot); } uptr internal_close(fd_t fd) { @@ -167,7 +192,7 @@ uptr internal_open(const char *filename, int flags) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags); #else - return internal_syscall(SYSCALL(open), (uptr)filename, flags); + return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags); #endif } @@ -176,32 +201,36 @@ uptr internal_open(const char *filename, int flags, u32 mode) { return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags, mode); #else - return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode); + return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags, mode); #endif } uptr internal_read(fd_t fd, void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(read), fd, (uptr)buf, + HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(read), fd, (uptr)buf, count)); return res; } uptr internal_write(fd_t fd, const void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(write), fd, (uptr)buf, + HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(write), fd, (uptr)buf, count)); return res; } uptr internal_ftruncate(fd_t fd, uptr size) { sptr res; +#if SANITIZER_NETBSD + HANDLE_EINTR(res, internal_syscall64(SYSCALL(ftruncate), fd, 0, (s64)size)); +#else HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd, (OFF_T)size)); +#endif return res; } -#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && !SANITIZER_FREEBSD +#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX static void stat64_to_stat(struct stat64 *in, struct stat *out) { internal_memset(out, 0, sizeof(*out)); out->st_dev = in->st_dev; @@ -221,6 +250,21 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) { #endif #if defined(__mips64) +// Undefine compatibility macros from <sys/stat.h> +// so that they would not clash with the kernel_stat +// st_[a|m|c]time fields +#undef st_atime +#undef st_mtime +#undef st_ctime +#if defined(SANITIZER_ANDROID) +// Bionic sys/stat.h defines additional macros +// for compatibility with the old NDKs and +// they clash with the kernel_stat structure +// st_[a|m|c]time_nsec fields. +#undef st_atime_nsec +#undef st_mtime_nsec +#undef st_ctime_nsec +#endif static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { internal_memset(out, 0, sizeof(*out)); out->st_dev = in->st_dev; @@ -233,15 +277,29 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { out->st_size = in->st_size; out->st_blksize = in->st_blksize; out->st_blocks = in->st_blocks; - out->st_atime = in->st_atime_nsec; - out->st_mtime = in->st_mtime_nsec; - out->st_ctime = in->st_ctime_nsec; +#if defined(__USE_MISC) || \ + defined(__USE_XOPEN2K8) || \ + defined(SANITIZER_ANDROID) + out->st_atim.tv_sec = in->st_atime; + out->st_atim.tv_nsec = in->st_atime_nsec; + out->st_mtim.tv_sec = in->st_mtime; + out->st_mtim.tv_nsec = in->st_mtime_nsec; + out->st_ctim.tv_sec = in->st_ctime; + out->st_ctim.tv_nsec = in->st_ctime_nsec; +#else + out->st_atime = in->st_atime; + out->st_atimensec = in->st_atime_nsec; + out->st_mtime = in->st_mtime; + out->st_mtimensec = in->st_mtime_nsec; + out->st_ctime = in->st_ctime; + out->st_atimensec = in->st_ctime_nsec; +#endif } #endif uptr internal_stat(const char *path, void *buf) { -#if SANITIZER_FREEBSD - return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, @@ -265,7 +323,9 @@ uptr internal_stat(const char *path, void *buf) { } uptr internal_lstat(const char *path, void *buf) { -#if SANITIZER_FREEBSD +#if SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(lstat), path, buf); +#elif SANITIZER_FREEBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS @@ -290,15 +350,15 @@ uptr internal_lstat(const char *path, void *buf) { } uptr internal_fstat(fd_t fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS -# if SANITIZER_MIPS64 +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +# if SANITIZER_MIPS64 && !SANITIZER_NETBSD // For mips64, fstat syscall fills buffer in the format of kernel_stat struct kernel_stat kbuf; int res = internal_syscall(SYSCALL(fstat), fd, &kbuf); kernel_stat_to_stat(&kbuf, (struct stat *)buf); return res; # else - return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); + return internal_syscall_ptr(SYSCALL(fstat), fd, (uptr)buf); # endif #else struct stat64 buf64; @@ -328,7 +388,7 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) { return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, bufsize); #else - return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize); + return internal_syscall_ptr(SYSCALL(readlink), path, buf, bufsize); #endif } @@ -336,7 +396,7 @@ uptr internal_unlink(const char *path) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0); #else - return internal_syscall(SYSCALL(unlink), (uptr)path); + return internal_syscall_ptr(SYSCALL(unlink), (uptr)path); #endif } @@ -345,7 +405,7 @@ uptr internal_rename(const char *oldpath, const char *newpath) { return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath); #else - return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); + return internal_syscall_ptr(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); #endif } @@ -354,7 +414,7 @@ uptr internal_sched_yield() { } void internal__exit(int exitcode) { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD internal_syscall(SYSCALL(exit), exitcode); #else internal_syscall(SYSCALL(exit_group), exitcode); @@ -366,16 +426,17 @@ unsigned int internal_sleep(unsigned int seconds) { struct timespec ts; ts.tv_sec = 1; ts.tv_nsec = 0; - int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts); + int res = internal_syscall_ptr(SYSCALL(nanosleep), &ts, &ts); if (res) return ts.tv_sec; return 0; } uptr internal_execve(const char *filename, char *const argv[], char *const envp[]) { - return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv, + return internal_syscall_ptr(SYSCALL(execve), (uptr)filename, (uptr)argv, (uptr)envp); } +#endif // !SANITIZER_SOLARIS // ----------------- sanitizer_common.h bool FileExists(const char *filename) { @@ -393,27 +454,37 @@ bool FileExists(const char *filename) { tid_t GetTid() { #if SANITIZER_FREEBSD return (uptr)pthread_self(); +#elif SANITIZER_NETBSD + return _lwp_self(); +#elif SANITIZER_SOLARIS + return (uptr)thr_self(); #else return internal_syscall(SYSCALL(gettid)); #endif } +#if !SANITIZER_SOLARIS u64 NanoTime() { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD timeval tv; #else kernel_timeval tv; #endif internal_memset(&tv, 0, sizeof(tv)); - internal_syscall(SYSCALL(gettimeofday), (uptr)&tv, 0); + internal_syscall_ptr(SYSCALL(gettimeofday), &tv, 0); return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000; } +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { + return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp); +} +#endif // !SANITIZER_SOLARIS + // Like getenv, but reads env directly from /proc (on Linux) or parses the -// 'environ' array (on FreeBSD) and does not use libc. This function should be -// called first inside __asan_init. +// 'environ' array (on some others) and does not use libc. This function +// should be called first inside __asan_init. const char *GetEnv(const char *name) { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_SOLARIS if (::environ != 0) { uptr NameLen = internal_strlen(name); for (char **Env = ::environ; *Env != 0; Env++) { @@ -451,13 +522,13 @@ const char *GetEnv(const char *name) { #endif } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD extern "C" { SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end; } #endif -#if !SANITIZER_GO && !SANITIZER_FREEBSD +#if !SANITIZER_GO && !SANITIZER_FREEBSD && !SANITIZER_NETBSD static void ReadNullSepFileToArray(const char *path, char ***arr, int arr_size) { char *buff; @@ -483,7 +554,22 @@ static void ReadNullSepFileToArray(const char *path, char ***arr, #endif static void GetArgsAndEnv(char ***argv, char ***envp) { -#if !SANITIZER_FREEBSD +#if SANITIZER_FREEBSD + // On FreeBSD, retrieving the argument and environment arrays is done via the + // kern.ps_strings sysctl, which returns a pointer to a structure containing + // this information. See also <sys/exec.h>. + ps_strings *pss; + size_t sz = sizeof(pss); + if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { + Printf("sysctl kern.ps_strings failed\n"); + Die(); + } + *argv = pss->ps_argvstr; + *envp = pss->ps_envstr; +#elif SANITIZER_NETBSD + *argv = __ps_strings->ps_argvstr; + *argv = __ps_strings->ps_envstr; +#else #if !SANITIZER_GO if (&__libc_stack_end) { #endif @@ -498,18 +584,6 @@ static void GetArgsAndEnv(char ***argv, char ***envp) { ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp); } #endif -#else - // On FreeBSD, retrieving the argument and environment arrays is done via the - // kern.ps_strings sysctl, which returns a pointer to a structure containing - // this information. See also <sys/exec.h>. - ps_strings *pss; - size_t sz = sizeof(pss); - if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { - Printf("sysctl kern.ps_strings failed\n"); - Die(); - } - *argv = pss->ps_argvstr; - *envp = pss->ps_envstr; #endif } @@ -521,14 +595,32 @@ char **GetArgv() { void ReExec() { char **argv, **envp; + const char *pathname = "/proc/self/exe"; + +#if SANITIZER_NETBSD + static const int name[] = { + CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME, + }; + char path[400]; + size_t len; + + len = sizeof(path); + if (sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1) + pathname = path; +#elif SANITIZER_SOLARIS + pathname = getexecname(); + CHECK_NE(pathname, NULL); +#endif + GetArgsAndEnv(&argv, &envp); - uptr rv = internal_execve("/proc/self/exe", argv, envp); + uptr rv = internal_execve(pathname, argv, envp); int rverrno; CHECK_EQ(internal_iserror(rv, &rverrno), true); Printf("execve failed, errno %d\n", rverrno); Die(); } +#if !SANITIZER_SOLARIS enum MutexState { MtxUnlocked = 0, MtxLocked = 1, @@ -547,6 +639,8 @@ void BlockingMutex::Lock() { while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) { #if SANITIZER_FREEBSD _umtx_op(m, UMTX_OP_WAIT_UINT, MtxSleeping, 0, 0); +#elif SANITIZER_NETBSD + sched_yield(); /* No userspace futex-like synchronization */ #else internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT, MtxSleeping, 0, 0, 0); #endif @@ -560,6 +654,8 @@ void BlockingMutex::Unlock() { if (v == MtxSleeping) { #if SANITIZER_FREEBSD _umtx_op(m, UMTX_OP_WAKE, 1, 0, 0); +#elif SANITIZER_NETBSD + /* No userspace futex-like synchronization */ #else internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE, 1, 0, 0, 0); #endif @@ -570,11 +666,23 @@ void BlockingMutex::CheckLocked() { atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed)); } +#endif // !SANITIZER_SOLARIS // ----------------- sanitizer_linux.h // The actual size of this structure is specified by d_reclen. // Note that getdents64 uses a different structure format. We only provide the // 32-bit syscall here. +#if SANITIZER_NETBSD +// struct dirent is different for Linux and us. At this moment, we use only +// d_fileno (Linux call this d_ino), d_reclen, and d_name. +struct linux_dirent { + u64 d_ino; // d_fileno + u16 d_reclen; + u16 d_namlen; // not used + u8 d_type; // not used + char d_name[NAME_MAX + 1]; +}; +#else struct linux_dirent { #if SANITIZER_X32 || defined(__aarch64__) u64 d_ino; @@ -589,15 +697,29 @@ struct linux_dirent { #endif char d_name[256]; }; +#endif +#if !SANITIZER_SOLARIS // Syscall wrappers. uptr internal_ptrace(int request, int pid, void *addr, void *data) { +#if SANITIZER_NETBSD + // XXX We need additional work for ptrace: + // - for request, we use PT_FOO whereas Linux uses PTRACE_FOO + // - data is int for us, but void * for Linux + // - Linux sometimes uses data in the case where we use addr instead + // At this moment, this function is used only within + // "#if SANITIZER_LINUX && defined(__x86_64__)" block in + // sanitizer_stoptheworld_linux_libcdep.cc. + return internal_syscall_ptr(SYSCALL(ptrace), request, pid, (uptr)addr, + (uptr)data); +#else return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr, (uptr)data); +#endif } uptr internal_waitpid(int pid, int *status, int options) { - return internal_syscall(SYSCALL(wait4), pid, (uptr)status, options, + return internal_syscall_ptr(SYSCALL(wait4), pid, (uptr)status, options, 0 /* rusage */); } @@ -615,12 +737,16 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) { #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count); #else - return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count); + return internal_syscall_ptr(SYSCALL(getdents), fd, (uptr)dirp, count); #endif } uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { +#if SANITIZER_NETBSD + return internal_syscall64(SYSCALL(lseek), fd, 0, offset, whence); +#else return internal_syscall(SYSCALL(lseek), fd, offset, whence); +#endif } #if SANITIZER_LINUX @@ -630,7 +756,7 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { #endif uptr internal_sigaltstack(const void *ss, void *oss) { - return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); + return internal_syscall_ptr(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); } int internal_fork() { @@ -711,8 +837,8 @@ int internal_sigaction_syscall(int signum, const void *act, void *oldact) { uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { -#if SANITIZER_FREEBSD - return internal_syscall(SYSCALL(sigprocmask), how, set, oldset); +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(sigprocmask), how, set, oldset); #else __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set; __sanitizer_kernel_sigset_t *k_oldset = (__sanitizer_kernel_sigset_t *)oldset; @@ -751,6 +877,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { return k_set->sig[idx] & (1 << bit); } #endif // SANITIZER_LINUX +#endif // !SANITIZER_SOLARIS // ThreadLister implementation. ThreadLister::ThreadLister(int pid) @@ -856,7 +983,9 @@ static uptr GetKernelAreaSize() { #endif // SANITIZER_WORDSIZE == 32 uptr GetMaxVirtualAddress() { -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_NETBSD && defined(__x86_64__) + return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE) +#elif SANITIZER_WORDSIZE == 64 # if defined(__powerpc64__) || defined(__aarch64__) // On PowerPC64 we have two different address space layouts: 44- and 46-bit. // We somehow need to figure out which one we are using now and choose @@ -877,15 +1006,21 @@ uptr GetMaxVirtualAddress() { # if defined(__s390__) return (1ULL << 31) - 1; // 0x7fffffff; # else - uptr res = (1ULL << 32) - 1; // 0xffffffff; - if (!common_flags()->full_address_space) - res -= GetKernelAreaSize(); - CHECK_LT(reinterpret_cast<uptr>(&res), res); - return res; + return (1ULL << 32) - 1; // 0xffffffff; # endif #endif // SANITIZER_WORDSIZE } +uptr GetMaxUserVirtualAddress() { + uptr addr = GetMaxVirtualAddress(); +#if SANITIZER_WORDSIZE == 32 && !defined(__s390__) + if (!common_flags()->full_address_space) + addr -= GetKernelAreaSize(); + CHECK_LT(reinterpret_cast<uptr>(&addr), addr); +#endif + return addr; +} + uptr GetPageSize() { // Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. #if SANITIZER_ANDROID @@ -900,8 +1035,17 @@ uptr GetPageSize() { } uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { +#if SANITIZER_SOLARIS + const char *default_module_name = getexecname(); + CHECK_NE(default_module_name, NULL); + return internal_snprintf(buf, buf_len, "%s", default_module_name); +#else +#if SANITIZER_FREEBSD || SANITIZER_NETBSD #if SANITIZER_FREEBSD - const int Mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; + const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; +#else + const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME}; +#endif const char *default_module_name = "kern.proc.pathname"; size_t Size = buf_len; bool IsErr = (sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0); @@ -913,7 +1057,7 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { default_module_name, buf, buf_len); int readlink_error; bool IsErr = internal_iserror(module_name_len, &readlink_error); -#endif +#endif // SANITIZER_SOLARIS if (IsErr) { // We can't read binary name for some reason, assume it's unknown. Report("WARNING: reading executable name failed with errno %d, " @@ -923,6 +1067,7 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { CHECK_LT(module_name_len, buf_len); } return module_name_len; +#endif } uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) { @@ -1539,12 +1684,17 @@ static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) { } #endif -SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { +SignalContext::WriteFlag SignalContext::GetWriteFlag() const { ucontext_t *ucontext = (ucontext_t *)context; #if defined(__x86_64__) || defined(__i386__) static const uptr PF_WRITE = 1U << 1; #if SANITIZER_FREEBSD uptr err = ucontext->uc_mcontext.mc_err; +#elif SANITIZER_NETBSD + uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR]; +#elif SANITIZER_SOLARIS && defined(__i386__) +# define ERR 13 + uptr err = ucontext->uc_mcontext.gregs[ERR]; #else uptr err = ucontext->uc_mcontext.gregs[REG_ERR]; #endif @@ -1558,6 +1708,12 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { u64 esr; if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN; return esr & ESR_ELx_WNR ? WRITE : READ; +#elif SANITIZER_SOLARIS && defined(__sparc__) + // Decode the instruction to determine the access type. + // From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype). + uptr pc = ucontext->uc_mcontext.gregs[REG_PC]; + u32 instr = *(u32 *)pc; + return (instr >> 21) & 1 ? WRITE: READ; #else (void)ucontext; return UNKNOWN; // FIXME: Implement. @@ -1568,7 +1724,7 @@ void SignalContext::DumpAllRegisters(void *context) { // FIXME: Implement this. } -void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { +static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #if defined(__arm__) ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.arm_pc; @@ -1591,6 +1747,11 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.mc_rip; *bp = ucontext->uc_mcontext.mc_rbp; *sp = ucontext->uc_mcontext.mc_rsp; +#elif SANITIZER_NETBSD + ucontext_t *ucontext = (ucontext_t *)context; + *pc = ucontext->uc_mcontext.__gregs[_REG_RIP]; + *bp = ucontext->uc_mcontext.__gregs[_REG_RBP]; + *sp = ucontext->uc_mcontext.__gregs[_REG_RSP]; # else ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.gregs[REG_RIP]; @@ -1603,8 +1764,26 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.mc_eip; *bp = ucontext->uc_mcontext.mc_ebp; *sp = ucontext->uc_mcontext.mc_esp; +#elif SANITIZER_NETBSD + ucontext_t *ucontext = (ucontext_t *)context; + *pc = ucontext->uc_mcontext.__gregs[_REG_EIP]; + *bp = ucontext->uc_mcontext.__gregs[_REG_EBP]; + *sp = ucontext->uc_mcontext.__gregs[_REG_ESP]; # else ucontext_t *ucontext = (ucontext_t*)context; +# if SANITIZER_SOLARIS + /* Use the numeric values: the symbolic ones are undefined by llvm + include/llvm/Support/Solaris.h. */ +# ifndef REG_EIP +# define REG_EIP 14 // REG_PC +# endif +# ifndef REG_EBP +# define REG_EBP 6 // REG_FP +# endif +# ifndef REG_ESP +# define REG_ESP 17 // REG_SP +# endif +# endif *pc = ucontext->uc_mcontext.gregs[REG_EIP]; *bp = ucontext->uc_mcontext.gregs[REG_EBP]; *sp = ucontext->uc_mcontext.gregs[REG_ESP]; @@ -1619,7 +1798,16 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #elif defined(__sparc__) ucontext_t *ucontext = (ucontext_t*)context; uptr *stk_ptr; -# if defined (__arch64__) +# if defined (__sparcv9) +# ifndef MC_PC +# define MC_PC REG_PC +# endif +# ifndef MC_O6 +# define MC_O6 REG_O6 +# endif +# ifdef SANITIZER_SOLARIS +# define mc_gregs gregs +# endif *pc = ucontext->uc_mcontext.mc_gregs[MC_PC]; *sp = ucontext->uc_mcontext.mc_gregs[MC_O6]; stk_ptr = (uptr *) (*sp + 2047); @@ -1649,6 +1837,8 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #endif } +void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } + void MaybeReexec() { // No need to re-exec on Linux. } @@ -1676,25 +1866,27 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding, return 0; } -bool GetRandom(void *buffer, uptr length) { +bool GetRandom(void *buffer, uptr length, bool blocking) { if (!buffer || !length || length > 256) return false; -#if defined(__NR_getrandom) +#if SANITIZER_USE_GETRANDOM static atomic_uint8_t skip_getrandom_syscall; if (!atomic_load_relaxed(&skip_getrandom_syscall)) { // Up to 256 bytes, getrandom will not be interrupted. - uptr res = internal_syscall(SYSCALL(getrandom), buffer, length, 0); + uptr res = internal_syscall(SYSCALL(getrandom), buffer, length, + blocking ? 0 : GRND_NONBLOCK); int rverrno = 0; if (internal_iserror(res, &rverrno) && rverrno == ENOSYS) atomic_store_relaxed(&skip_getrandom_syscall, 1); else if (res == length) return true; } -#endif +#endif // SANITIZER_USE_GETRANDOM + // Up to 256 bytes, a read off /dev/urandom will not be interrupted. + // blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom. uptr fd = internal_open("/dev/urandom", O_RDONLY); if (internal_iserror(fd)) return false; - // internal_read deals with EINTR. uptr res = internal_read(fd, buffer, length); if (internal_iserror(res)) return false; @@ -1704,4 +1896,6 @@ bool GetRandom(void *buffer, uptr length) { } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS + diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h index 11cad6b80933..2d227f82a99e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h @@ -14,11 +14,14 @@ #define SANITIZER_LINUX_H #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" -#include "sanitizer_posix.h" +#include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_posix.h" +#include "sanitizer_platform_limits_solaris.h" +#include "sanitizer_posix.h" struct link_map; // Opaque type returned by dlopen(). @@ -27,11 +30,25 @@ namespace __sanitizer { // the one in <dirent.h>, which is used by readdir(). struct linux_dirent; +struct ProcSelfMapsBuff { + char *data; + uptr mmaped_size; + uptr len; +}; + +struct MemoryMappingLayoutData { + ProcSelfMapsBuff proc_self_maps; + const char *current; +}; + +void ReadProcMaps(ProcSelfMapsBuff *proc_maps); + // Syscall wrappers. uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count); uptr internal_sigaltstack(const void* ss, void* oss); uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp); // Linux-only syscalls. #if SANITIZER_LINUX @@ -128,5 +145,6 @@ ALWAYS_INLINE uptr *get_android_tls_ptr() { } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS #endif // SANITIZER_LINUX_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 52196db12731..56fdfc8705f3 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -14,11 +14,13 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_allocator_internal.h" #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_freebsd.h" #include "sanitizer_linux.h" @@ -36,19 +38,35 @@ #if SANITIZER_FREEBSD #include <pthread_np.h> #include <osreldate.h> +#include <sys/sysctl.h> #define pthread_getattr_np pthread_attr_get_np #endif +#if SANITIZER_NETBSD +#include <sys/sysctl.h> +#include <sys/tls.h> +#endif + +#if SANITIZER_SOLARIS +#include <thread.h> +#endif + #if SANITIZER_LINUX #include <sys/prctl.h> #endif #if SANITIZER_ANDROID #include <android/api-level.h> +#if !defined(CPU_COUNT) && !defined(__aarch64__) +#include <dirent.h> +#include <fcntl.h> +struct __sanitizer::linux_dirent { + long d_ino; + off_t d_off; + unsigned short d_reclen; + char d_name[]; +}; #endif - -#if SANITIZER_ANDROID && __ANDROID_API__ < 21 -#include <android/log.h> #endif #if !SANITIZER_ANDROID @@ -102,13 +120,20 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, *stack_bottom = segment.end - stacksize; return; } + uptr stacksize = 0; + void *stackaddr = nullptr; +#if SANITIZER_SOLARIS + stack_t ss; + CHECK_EQ(thr_stksegment(&ss), 0); + stacksize = ss.ss_size; + stackaddr = (char *)ss.ss_sp - stacksize; +#else // !SANITIZER_SOLARIS pthread_attr_t attr; pthread_attr_init(&attr); CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); - uptr stacksize = 0; - void *stackaddr = nullptr; my_pthread_attr_getstack(&attr, &stackaddr, &stacksize); pthread_attr_destroy(&attr); +#endif // SANITIZER_SOLARIS *stack_top = (uptr)stackaddr + stacksize; *stack_bottom = (uptr)stackaddr; @@ -148,7 +173,8 @@ bool SanitizerGetThreadName(char *name, int max_len) { #endif } -#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \ + !SANITIZER_NETBSD && !SANITIZER_SOLARIS static uptr g_tls_size; #ifdef __i386__ @@ -176,7 +202,8 @@ void InitTlsSize() { } #else void InitTlsSize() { } -#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO +#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && + // !SANITIZER_NETBSD && !SANITIZER_SOLARIS #if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) \ || defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) \ @@ -323,7 +350,7 @@ static void **ThreadSelfSegbase() { // sysarch(AMD64_GET_FSBASE, segbase); __asm __volatile("movq %%fs:0, %0" : "=r" (segbase)); # else -# error "unsupported CPU arch for FreeBSD platform" +# error "unsupported CPU arch" # endif return segbase; } @@ -333,6 +360,35 @@ uptr ThreadSelf() { } #endif // SANITIZER_FREEBSD +#if SANITIZER_NETBSD +static struct tls_tcb * ThreadSelfTlsTcb() { + struct tls_tcb * tcb; +# ifdef __HAVE___LWP_GETTCB_FAST + tcb = (struct tls_tcb *)__lwp_gettcb_fast(); +# elif defined(__HAVE___LWP_GETPRIVATE_FAST) + tcb = (struct tls_tcb *)__lwp_getprivate_fast(); +# endif + return tcb; +} + +uptr ThreadSelf() { + return (uptr)ThreadSelfTlsTcb()->tcb_pthread; +} + +int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) { + const Elf_Phdr *hdr = info->dlpi_phdr; + const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum; + + for (; hdr != last_hdr; ++hdr) { + if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) { + *(uptr*)data = hdr->p_memsz; + break; + } + } + return 0; +} +#endif // SANITIZER_NETBSD + #if !SANITIZER_GO static void GetTls(uptr *addr, uptr *size) { #if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -362,9 +418,27 @@ static void GetTls(uptr *addr, uptr *size) { *addr = (uptr) dtv[2]; *size = (*addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]); } +#elif SANITIZER_NETBSD + struct tls_tcb * const tcb = ThreadSelfTlsTcb(); + *addr = 0; + *size = 0; + if (tcb != 0) { + // Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program). + // ld.elf_so hardcodes the index 1. + dl_iterate_phdr(GetSizeFromHdr, size); + + if (*size != 0) { + // The block has been found and tcb_dtv[1] contains the base address + *addr = (uptr)tcb->tcb_dtv[1]; + } + } #elif SANITIZER_ANDROID *addr = 0; *size = 0; +#elif SANITIZER_SOLARIS + // FIXME + *addr = 0; + *size = 0; #else # error "Unknown OS" #endif @@ -373,7 +447,8 @@ static void GetTls(uptr *addr, uptr *size) { #if !SANITIZER_GO uptr GetTlsSize() { -#if SANITIZER_FREEBSD || SANITIZER_ANDROID +#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS uptr addr, size; GetTls(&addr, &size); return size; @@ -419,7 +494,7 @@ typedef ElfW(Phdr) Elf_Phdr; # endif struct DlIteratePhdrData { - InternalMmapVector<LoadedModule> *modules; + InternalMmapVectorNoCtor<LoadedModule> *modules; bool first; }; @@ -457,21 +532,41 @@ extern "C" __attribute__((weak)) int dl_iterate_phdr( int (*)(struct dl_phdr_info *, size_t, void *), void *); #endif -void ListOfModules::init() { - clear(); +static bool requiresProcmaps() { #if SANITIZER_ANDROID && __ANDROID_API__ <= 22 - u32 api_level = AndroidGetApiLevel(); // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken. // The runtime check allows the same library to work with // both K and L (and future) Android releases. - if (api_level <= ANDROID_LOLLIPOP_MR1) { // L or earlier - MemoryMappingLayout memory_mapping(false); - memory_mapping.DumpListOfModules(&modules_); - return; - } + return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1; +#else + return false; #endif - DlIteratePhdrData data = {&modules_, true}; - dl_iterate_phdr(dl_iterate_phdr_cb, &data); +} + +static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) { + MemoryMappingLayout memory_mapping(/*cache_enabled*/true); + memory_mapping.DumpListOfModules(modules); +} + +void ListOfModules::init() { + clearOrInit(); + if (requiresProcmaps()) { + procmapsInit(&modules_); + } else { + DlIteratePhdrData data = {&modules_, true}; + dl_iterate_phdr(dl_iterate_phdr_cb, &data); + } +} + +// When a custom loader is used, dl_iterate_phdr may not contain the full +// list of modules. Allow callers to fall back to using procmaps. +void ListOfModules::fallbackInit() { + if (!requiresProcmaps()) { + clearOrInit(); + procmapsInit(&modules_); + } else { + clear(); + } } // getrusage does not give us the current RSS, only the max RSS. @@ -513,12 +608,65 @@ uptr GetRSS() { return rss * GetPageSizeCached(); } -// 64-bit Android targets don't provide the deprecated __android_log_write. -// Starting with the L release, syslog() works and is preferable to -// __android_log_write. +// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as +// they allocate memory. +u32 GetNumberOfCPUs() { +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + u32 ncpu; + int req[2]; + size_t len = sizeof(ncpu); + req[0] = CTL_HW; + req[1] = HW_NCPU; + CHECK_EQ(sysctl(req, 2, &ncpu, &len, NULL, 0), 0); + return ncpu; +#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__) + // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't + // exist in sched.h. That is the case for toolchains generated with older + // NDKs. + // This code doesn't work on AArch64 because internal_getdents makes use of + // the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64. + uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY | O_DIRECTORY); + if (internal_iserror(fd)) + return 0; + InternalScopedBuffer<u8> buffer(4096); + uptr bytes_read = buffer.size(); + uptr n_cpus = 0; + u8 *d_type; + struct linux_dirent *entry = (struct linux_dirent *)&buffer[bytes_read]; + while (true) { + if ((u8 *)entry >= &buffer[bytes_read]) { + bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(), + buffer.size()); + if (internal_iserror(bytes_read) || !bytes_read) + break; + entry = (struct linux_dirent *)buffer.data(); + } + d_type = (u8 *)entry + entry->d_reclen - 1; + if (d_type >= &buffer[bytes_read] || + (u8 *)&entry->d_name[3] >= &buffer[bytes_read]) + break; + if (entry->d_ino != 0 && *d_type == DT_DIR) { + if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' && + entry->d_name[2] == 'u' && + entry->d_name[3] >= '0' && entry->d_name[3] <= '9') + n_cpus++; + } + entry = (struct linux_dirent *)(((u8 *)entry) + entry->d_reclen); + } + internal_close(fd); + return n_cpus; +#elif SANITIZER_SOLARIS + return sysconf(_SC_NPROCESSORS_ONLN); +#else + cpu_set_t CPUs; + CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); + return CPU_COUNT(&CPUs); +#endif +} + #if SANITIZER_LINUX -#if SANITIZER_ANDROID +# if SANITIZER_ANDROID static atomic_uint8_t android_log_initialized; void AndroidLogInit() { @@ -529,34 +677,97 @@ void AndroidLogInit() { static bool ShouldLogAfterPrintf() { return atomic_load(&android_log_initialized, memory_order_acquire); } -#else -void AndroidLogInit() {} -static bool ShouldLogAfterPrintf() { return true; } -#endif // SANITIZER_ANDROID +extern "C" SANITIZER_WEAK_ATTRIBUTE +int async_safe_write_log(int pri, const char* tag, const char* msg); +extern "C" SANITIZER_WEAK_ATTRIBUTE +int __android_log_write(int prio, const char* tag, const char* msg); + +// ANDROID_LOG_INFO is 4, but can't be resolved at runtime. +#define SANITIZER_ANDROID_LOG_INFO 4 +// async_safe_write_log is a new public version of __libc_write_log that is +// used behind syslog. It is preferable to syslog as it will not do any dynamic +// memory allocation or formatting. +// If the function is not available, syslog is preferred for L+ (it was broken +// pre-L) as __android_log_write triggers a racey behavior with the strncpy +// interceptor. Fallback to __android_log_write pre-L. void WriteOneLineToSyslog(const char *s) { -#if SANITIZER_ANDROID &&__ANDROID_API__ < 21 - __android_log_write(ANDROID_LOG_INFO, NULL, s); -#else - syslog(LOG_INFO, "%s", s); -#endif + if (&async_safe_write_log) { + async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s); + } else if (AndroidGetApiLevel() > ANDROID_KITKAT) { + syslog(LOG_INFO, "%s", s); + } else { + CHECK(&__android_log_write); + __android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s); + } } +extern "C" SANITIZER_WEAK_ATTRIBUTE +void android_set_abort_message(const char *); + +void SetAbortMessage(const char *str) { + if (&android_set_abort_message) + android_set_abort_message(str); +} +# else +void AndroidLogInit() {} + +static bool ShouldLogAfterPrintf() { return true; } + +void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); } + +void SetAbortMessage(const char *str) {} +# endif // SANITIZER_ANDROID + void LogMessageOnPrintf(const char *str) { if (common_flags()->log_to_syslog && ShouldLogAfterPrintf()) WriteToSyslog(str); } -#if SANITIZER_ANDROID && __ANDROID_API__ >= 21 -extern "C" void android_set_abort_message(const char *msg); -void SetAbortMessage(const char *str) { android_set_abort_message(str); } -#else -void SetAbortMessage(const char *str) {} -#endif +#endif // SANITIZER_LINUX + +#if SANITIZER_LINUX && !SANITIZER_GO +// glibc crashes when using clock_gettime from a preinit_array function as the +// vDSO function pointers haven't been initialized yet. __progname is +// initialized after the vDSO function pointers, so if it exists, is not null +// and is not empty, we can use clock_gettime. +extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname; +INLINE bool CanUseVDSO() { + // Bionic is safe, it checks for the vDSO function pointers to be initialized. + if (SANITIZER_ANDROID) + return true; + if (&__progname && __progname && *__progname) + return true; + return false; +} -#endif // SANITIZER_LINUX +// MonotonicNanoTime is a timing function that can leverage the vDSO by calling +// clock_gettime. real_clock_gettime only exists if clock_gettime is +// intercepted, so define it weakly and use it if available. +extern "C" SANITIZER_WEAK_ATTRIBUTE +int real_clock_gettime(u32 clk_id, void *tp); +u64 MonotonicNanoTime() { + timespec ts; + if (CanUseVDSO()) { + if (&real_clock_gettime) + real_clock_gettime(CLOCK_MONOTONIC, &ts); + else + clock_gettime(CLOCK_MONOTONIC, &ts); + } else { + internal_clock_gettime(CLOCK_MONOTONIC, &ts); + } + return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec; +} +#else +// Non-Linux & Go always use the syscall. +u64 MonotonicNanoTime() { + timespec ts; + internal_clock_gettime(CLOCK_MONOTONIC, &ts); + return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec; +} +#endif // SANITIZER_LINUX && !SANITIZER_GO } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc index 1edd4157fd6b..e1c51f580874 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc @@ -23,6 +23,7 @@ #include <stdio.h> #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" @@ -184,7 +185,7 @@ uptr internal_getpid() { int internal_sigaction(int signum, const void *act, void *oldact) { return sigaction(signum, - (struct sigaction *)act, (struct sigaction *)oldact); + (const struct sigaction *)act, (struct sigaction *)oldact); } void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); } @@ -364,6 +365,10 @@ u64 NanoTime() { return 0; } +u64 MonotonicNanoTime() { + return 0; +} + uptr GetTlsSize() { return 0; } @@ -410,11 +415,13 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, } void ListOfModules::init() { - clear(); + clearOrInit(); MemoryMappingLayout memory_mapping(false); memory_mapping.DumpListOfModules(&modules_); } +void ListOfModules::fallbackInit() { clear(); } + static HandleSignalMode GetHandleSignalModeImpl(int signum) { switch (signum) { case SIGABRT: @@ -573,7 +580,7 @@ void LogFullErrorReport(const char *buffer) { #endif } -SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { +SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if defined(__x86_64__) || defined(__i386__) ucontext_t *ucontext = static_cast<ucontext_t*>(context); return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ; @@ -582,7 +589,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { #endif } -void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { +static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { ucontext_t *ucontext = (ucontext_t*)context; # if defined(__aarch64__) *pc = ucontext->uc_mcontext->__ss.__pc; @@ -609,6 +616,8 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { # endif } +void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } + #if !SANITIZER_GO static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; LowLevelAllocator allocator_for_env; @@ -736,6 +745,9 @@ void MaybeReexec() { if (!lib_is_in_env) return; + if (!common_flags()->strip_env) + return; + // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove // the dylib from the environment variable, because interceptors are installed // and we don't want our children to inherit the variable. @@ -844,7 +856,7 @@ uptr GetTaskInfoMaxAddress() { } #endif -uptr GetMaxVirtualAddress() { +uptr GetMaxUserVirtualAddress() { #if SANITIZER_WORDSIZE == 64 # if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM // Get the maximum VM address @@ -859,6 +871,10 @@ uptr GetMaxVirtualAddress() { #endif // SANITIZER_WORDSIZE } +uptr GetMaxVirtualAddress() { + return GetMaxUserVirtualAddress(); +} + uptr FindAvailableMemoryRange(uptr shadow_size, uptr alignment, uptr left_padding, @@ -881,6 +897,11 @@ uptr FindAvailableMemoryRange(uptr shadow_size, mach_msg_type_number_t count = kRegionInfoSize; kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth, (vm_region_info_t)&vminfo, &count); + if (kr == KERN_INVALID_ADDRESS) { + // No more regions beyond "address", consider the gap at the end of VM. + address = GetMaxVirtualAddress() + 1; + vmsize = 0; + } if (free_begin != address) { // We found a free region [free_begin..address-1]. uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment); @@ -991,7 +1012,12 @@ void CheckNoDeepBind(const char *filename, int flag) { } // FIXME: implement on this platform. -bool GetRandom(void *buffer, uptr length) { +bool GetRandom(void *buffer, uptr length, bool blocking) { + UNIMPLEMENTED(); +} + +// FIXME: implement on this platform. +u32 GetNumberOfCPUs() { UNIMPLEMENTED(); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h index 3f1c68c8610a..e022a2c0363c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h @@ -20,6 +20,17 @@ namespace __sanitizer { +struct MemoryMappingLayoutData { + int current_image; + u32 current_magic; + u32 current_filetype; + ModuleArch current_arch; + u8 current_uuid[kModuleUUIDSize]; + int current_load_cmd_count; + const char *current_load_cmd_addr; + bool current_instrumented; +}; + enum MacosVersion { MACOS_VERSION_UNINITIALIZED = 0, MACOS_VERSION_UNKNOWN, diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc index c95daa9372ad..090f17c888b4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc @@ -20,7 +20,7 @@ namespace __sanitizer { void RestrictMemoryToMaxAddress(uptr max_address) { - uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address; + uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address; void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap"); CHECK(res != MAP_FAILED); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h index 1759bf13f689..4bb878ee19dc 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h @@ -92,8 +92,10 @@ class BlockingMutex { // checks that the mutex is owned, and assumes callers to be generally // well-behaved. void CheckLocked(); + private: - uptr opaque_storage_[10]; + // Solaris mutex_t has a member that requires 64-bit alignment. + ALIGNED(8) uptr opaque_storage_[10]; uptr owner_; // for debugging }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h index 396f7c9346d6..334903c26f9f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -14,7 +14,8 @@ #define SANITIZER_PLATFORM_H #if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \ - !defined(__APPLE__) && !defined(_WIN32) + !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \ + !(defined(__sun__) && defined(__svr4__)) # error "This operating system is not supported" #endif @@ -36,6 +37,12 @@ # define SANITIZER_NETBSD 0 #endif +#if defined(__sun__) && defined(__svr4__) +# define SANITIZER_SOLARIS 1 +#else +# define SANITIZER_SOLARIS 0 +#endif + #if defined(__APPLE__) # define SANITIZER_MAC 1 # include <TargetConditionals.h> @@ -44,7 +51,7 @@ # else # define SANITIZER_IOS 0 # endif -# if TARGET_IPHONE_SIMULATOR +# if TARGET_OS_SIMULATOR # define SANITIZER_IOSSIM 1 # else # define SANITIZER_IOSSIM 0 @@ -85,8 +92,15 @@ # define SANITIZER_ANDROID 0 #endif +#if defined(__Fuchsia__) +# define SANITIZER_FUCHSIA 1 +#else +# define SANITIZER_FUCHSIA 0 +#endif + #define SANITIZER_POSIX \ - (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD) + (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \ + SANITIZER_NETBSD || SANITIZER_SOLARIS) #if __LP64__ || defined(_WIN64) # define SANITIZER_WORDSIZE 64 @@ -175,13 +189,19 @@ # define SANITIZER_ARM 0 #endif +#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32 +# define SANITIZER_SOLARIS32 1 +#else +# define SANITIZER_SOLARIS32 0 +#endif + // By default we allow to use SizeClassAllocator64 on 64-bit platform. // But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64 // does not work well and we need to fallback to SizeClassAllocator32. // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here. #ifndef SANITIZER_CAN_USE_ALLOCATOR64 -# if SANITIZER_ANDROID && defined(__aarch64__) +# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA # define SANITIZER_CAN_USE_ALLOCATOR64 1 # elif defined(__mips64) || defined(__aarch64__) # define SANITIZER_CAN_USE_ALLOCATOR64 0 @@ -276,5 +296,10 @@ # define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0 #endif +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS +# define SANITIZER_MADVISE_DONTNEED MADV_FREE +#else +# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED +#endif #endif // SANITIZER_PLATFORM_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h index 0380cee92a00..7e33fa59b312 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -16,19 +16,26 @@ #include "sanitizer_internal_defs.h" +#if SANITIZER_POSIX +# define SI_POSIX 1 +#else +# define SI_POSIX 0 +#endif + #if !SANITIZER_WINDOWS # define SI_WINDOWS 0 -# define SI_NOT_WINDOWS 1 -# include "sanitizer_platform_limits_posix.h" #else # define SI_WINDOWS 1 -# define SI_NOT_WINDOWS 0 #endif -#if SANITIZER_POSIX -# define SI_POSIX 1 -#else -# define SI_POSIX 0 +#if (SI_POSIX != 0) == (SI_WINDOWS != 0) && !SANITIZER_FUCHSIA +# error "Windows is not POSIX!" +#endif + +#if SI_POSIX +# include "sanitizer_platform_limits_netbsd.h" +# include "sanitizer_platform_limits_posix.h" +# include "sanitizer_platform_limits_solaris.h" #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -75,10 +82,28 @@ # define SI_IOS 0 #endif -#if !SANITIZER_WINDOWS && !SANITIZER_MAC -# define SI_UNIX_NOT_MAC 1 +#if SANITIZER_FUCHSIA +# define SI_NOT_FUCHSIA 0 +#else +# define SI_NOT_FUCHSIA 1 +#endif + +#if SANITIZER_SOLARIS +# define SI_SOLARIS 1 #else -# define SI_UNIX_NOT_MAC 0 +# define SI_SOLARIS 0 +#endif + +#if SANITIZER_SOLARIS32 +# define SI_SOLARIS32 1 +#else +# define SI_SOLARIS32 0 +#endif + +#if SANITIZER_POSIX && !SANITIZER_MAC +# define SI_POSIX_NOT_MAC 1 +#else +# define SI_POSIX_NOT_MAC 0 #endif #if SANITIZER_LINUX && !SANITIZER_FREEBSD @@ -87,23 +112,23 @@ # define SI_LINUX_NOT_FREEBSD 0 #endif -#define SANITIZER_INTERCEPT_STRLEN 1 -#define SANITIZER_INTERCEPT_STRNLEN SI_NOT_MAC -#define SANITIZER_INTERCEPT_STRCMP 1 -#define SANITIZER_INTERCEPT_STRSTR 1 -#define SANITIZER_INTERCEPT_STRCASESTR SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_STRTOK 1 -#define SANITIZER_INTERCEPT_STRCHR 1 -#define SANITIZER_INTERCEPT_STRCHRNUL SI_UNIX_NOT_MAC -#define SANITIZER_INTERCEPT_STRRCHR 1 -#define SANITIZER_INTERCEPT_STRSPN 1 -#define SANITIZER_INTERCEPT_STRPBRK 1 -#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STRCASECMP SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRNLEN (SI_NOT_MAC && SI_NOT_FUCHSIA) +#define SANITIZER_INTERCEPT_STRCMP SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRSTR SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRCASESTR SI_POSIX +#define SANITIZER_INTERCEPT_STRTOK SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRCHR SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRCHRNUL SI_POSIX_NOT_MAC +#define SANITIZER_INTERCEPT_STRRCHR SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRSPN SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRPBRK SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_STRCASECMP SI_POSIX #define SANITIZER_INTERCEPT_MEMSET 1 #define SANITIZER_INTERCEPT_MEMMOVE 1 #define SANITIZER_INTERCEPT_MEMCPY 1 -#define SANITIZER_INTERCEPT_MEMCMP 1 +#define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA #define SANITIZER_INTERCEPT_STRNDUP SI_POSIX #define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD #if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ @@ -114,24 +139,23 @@ #endif // memmem on Darwin doesn't exist on 10.6 // FIXME: enable memmem on Windows. -#define SANITIZER_INTERCEPT_MEMMEM \ - (SI_NOT_WINDOWS && !SI_MAC_DEPLOYMENT_BELOW_10_7) -#define SANITIZER_INTERCEPT_MEMCHR 1 +#define SANITIZER_INTERCEPT_MEMMEM (SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_7) +#define SANITIZER_INTERCEPT_MEMCHR SI_NOT_FUCHSIA #define SANITIZER_INTERCEPT_MEMRCHR (SI_FREEBSD || SI_LINUX || SI_NETBSD) -#define SANITIZER_INTERCEPT_READ SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PREAD SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_WRITE SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PWRITE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_READ SI_POSIX +#define SANITIZER_INTERCEPT_PREAD SI_POSIX +#define SANITIZER_INTERCEPT_WRITE SI_POSIX +#define SANITIZER_INTERCEPT_PWRITE SI_POSIX -#define SANITIZER_INTERCEPT_FREAD SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_FWRITE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_FREAD SI_POSIX +#define SANITIZER_INTERCEPT_FWRITE SI_POSIX -#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 +#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 -#define SANITIZER_INTERCEPT_READV SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_WRITEV SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_READV SI_POSIX +#define SANITIZER_INTERCEPT_WRITEV SI_POSIX #define SANITIZER_INTERCEPT_PREADV \ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) @@ -141,58 +165,68 @@ #define SANITIZER_INTERCEPT_PRCTL SI_LINUX -#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_STRPTIME SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_POSIX +#define SANITIZER_INTERCEPT_STRPTIME SI_POSIX -#define SANITIZER_INTERCEPT_SCANF SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SCANF SI_POSIX #define SANITIZER_INTERCEPT_ISOC99_SCANF SI_LINUX_NOT_ANDROID #ifndef SANITIZER_INTERCEPT_PRINTF -# define SANITIZER_INTERCEPT_PRINTF SI_NOT_WINDOWS +# define SANITIZER_INTERCEPT_PRINTF SI_POSIX # define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD) # define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_LINUX_NOT_ANDROID #endif -#define SANITIZER_INTERCEPT_FREXP 1 -#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT___PRINTF_CHK \ + (SANITIZER_INTERCEPT_PRINTF && SI_LINUX_NOT_ANDROID) + +#define SANITIZER_INTERCEPT_FREXP SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_POSIX -#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_POSIX #define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETPWENT \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_GETPWENT_R \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SETPWENT (SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_CLOCK_GETTIME (SI_FREEBSD || SI_NETBSD || SI_LINUX) -#define SANITIZER_INTERCEPT_GETITIMER SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_TIME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_WAIT SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_INET SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETADDRINFO SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETNAMEINFO SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETSOCKNAME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R (SI_FREEBSD || SI_LINUX) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_SETPWENT \ + (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_CLOCK_GETTIME \ + (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS) +#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX +#define SANITIZER_INTERCEPT_TIME SI_POSIX +#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_GLOB64 SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_WAIT SI_POSIX +#define SANITIZER_INTERCEPT_INET SI_POSIX +#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX +#define SANITIZER_INTERCEPT_GETADDRINFO SI_POSIX +#define SANITIZER_INTERCEPT_GETNAMEINFO SI_POSIX +#define SANITIZER_INTERCEPT_GETSOCKNAME SI_POSIX +#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_POSIX +#define SANITIZER_INTERCEPT_GETHOSTBYNAME2 SI_POSIX && !SI_SOLARIS +#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R \ + (SI_FREEBSD || SI_LINUX || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R \ (SI_FREEBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_GETSOCKOPT SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_ACCEPT SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R \ + (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_GETHOSTENT_R \ + (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX +#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX #define SANITIZER_INTERCEPT_ACCEPT4 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_MODF SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_RECVMSG SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SENDMSG SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETPEERNAME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_IOCTL SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_INET_ATON SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_MODF SI_POSIX +#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX +#define SANITIZER_INTERCEPT_SENDMSG SI_POSIX +#define SANITIZER_INTERCEPT_GETPEERNAME SI_POSIX +#define SANITIZER_INTERCEPT_IOCTL SI_POSIX +#define SANITIZER_INTERCEPT_INET_ATON SI_POSIX #define SANITIZER_INTERCEPT_SYSINFO SI_LINUX -#define SANITIZER_INTERCEPT_READDIR SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_READDIR SI_POSIX +#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 #if SI_LINUX_NOT_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ @@ -201,116 +235,125 @@ #else #define SANITIZER_INTERCEPT_PTRACE 0 #endif -#define SANITIZER_INTERCEPT_SETLOCALE SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETCWD SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SETLOCALE SI_POSIX +#define SANITIZER_INTERCEPT_GETCWD SI_POSIX #define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STRTOIMAX SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_MBSTOWCS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_MBSNRTOWCS (SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_WCSTOMBS SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_STRTOIMAX SI_POSIX +#define SANITIZER_INTERCEPT_MBSTOWCS SI_POSIX +#define SANITIZER_INTERCEPT_MBSNRTOWCS \ + (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_WCSTOMBS SI_POSIX #define SANITIZER_INTERCEPT_WCSNRTOMBS \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_WCRTOMB \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_REALPATH SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_REALPATH SI_POSIX +#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME \ + (SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_CONFSTR \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STRERROR SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_STRERROR_R SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_STRERROR SI_POSIX +#define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX #define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_SCANDIR \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_GETGROUPS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_POLL SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 +#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX +#define SANITIZER_INTERCEPT_POLL SI_POSIX +#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_WORDEXP \ - (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SIGWAIT SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \ + SI_SOLARIS) +#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX +#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_SIGSETOPS \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SIGPENDING SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SIGPROCMASK SI_NOT_WINDOWS + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX +#define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX #define SANITIZER_INTERCEPT_BACKTRACE \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX #define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STATFS \ - (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_STATFS64 \ ((SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_STATVFS \ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_INITGROUPS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX +#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX #define SANITIZER_INTERCEPT_ETHER_HOST \ (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_SHMCTL \ - (SI_NETBSD || ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && \ + (SI_NETBSD || SI_SOLARIS || ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && \ SANITIZER_WORDSIZE == 64)) // NOLINT #define SANITIZER_INTERCEPT_RANDOM_R SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \ + (SI_POSIX && !SI_NETBSD) +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL \ - (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING \ - (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST SI_LINUX_NOT_ANDROID + (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST \ + (SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED \ + (SI_POSIX && !SI_NETBSD) #define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_TMPNAM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_TTYNAME_R SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_TEMPNAM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SINCOS SI_LINUX -#define SANITIZER_INTERCEPT_REMQUO SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_LGAMMA SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX) -#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED \ + (SI_POSIX && !SI_NETBSD) +#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK \ + (SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \ + (SI_LINUX_NOT_ANDROID && !SI_NETBSD) +#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX +#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX +#define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX +#define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS +#define SANITIZER_INTERCEPT_REMQUO SI_POSIX +#define SANITIZER_INTERCEPT_LGAMMA SI_POSIX +#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS) +#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_RAND_R \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_ICONV \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_TIMES SI_NOT_WINDOWS + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_TIMES SI_POSIX // FIXME: getline seems to be available on OSX 10.7 #define SANITIZER_INTERCEPT_GETLINE \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT__EXIT \ - (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC) + (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC || SI_SOLARIS) -#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_TLS_GET_ADDR \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX #define SANITIZER_INTERCEPT_GETXATTR SI_LINUX #define SANITIZER_INTERCEPT_GETRESID SI_LINUX #define SANITIZER_INTERCEPT_GETIFADDRS \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS) #define SANITIZER_INTERCEPT_IF_INDEXTONAME \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS) #define SANITIZER_INTERCEPT_CAPGET SI_LINUX_NOT_ANDROID #if SI_LINUX && defined(__arm__) #define SANITIZER_INTERCEPT_AEABI_MEM 1 @@ -318,64 +361,71 @@ #define SANITIZER_INTERCEPT_AEABI_MEM 0 #endif #define SANITIZER_INTERCEPT___BZERO SI_MAC -#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_NOT_WINDOWS) -#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_POSIX) +#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_TSEARCH \ - (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD) + (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_FOPEN SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM (SI_LINUX_NOT_ANDROID || SI_NETBSD) +#define SANITIZER_INTERCEPT_FOPEN SI_POSIX +#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 +#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM \ + (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_FFLUSH SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_FCLOSE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_FFLUSH SI_POSIX +#define SANITIZER_INTERCEPT_FCLOSE SI_POSIX #ifndef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE #define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS) #endif #define SANITIZER_INTERCEPT_GETPASS \ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD) #define SANITIZER_INTERCEPT_TIMERFD SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_MLOCKX SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_MLOCKX SI_POSIX #define SANITIZER_INTERCEPT_FOPENCOOKIE SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_SEM (SI_LINUX || SI_FREEBSD || SI_NETBSD) -#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD) +#define SANITIZER_INTERCEPT_SEM \ + (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_SOLARIS) +#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX +#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX #define SANITIZER_INTERCEPT_CTERMID \ - (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD) -#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD) + (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_SOLARIS) +#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD || SI_SOLARIS) #define SANITIZER_INTERCEPTOR_HOOKS (SI_LINUX || SI_MAC || SI_WINDOWS) -#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SEND_SENDTO SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX +#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX #define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX #define SANITIZER_INTERCEPT_STAT \ - (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD) -#define SANITIZER_INTERCEPT___XSTAT \ - (!SANITIZER_INTERCEPT_STAT && SI_NOT_WINDOWS) + (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS) +#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX) #define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT #define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_UTMP (SI_NOT_WINDOWS && !SI_MAC && !SI_FREEBSD) -#define SANITIZER_INTERCEPT_UTMPX (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD) +#define SANITIZER_INTERCEPT_UTMP \ + (SI_POSIX && !SI_MAC && !SI_FREEBSD && !SI_NETBSD) +#define SANITIZER_INTERCEPT_UTMPX \ + (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD) #define SANITIZER_INTERCEPT_GETLOADAVG \ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD) #define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO \ - (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) + (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA) #define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) -#define SANITIZER_INTERCEPT_PVALLOC (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) -#define SANITIZER_INTERCEPT_CFREE (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) +#define SANITIZER_INTERCEPT_PVALLOC \ + (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA) +#define SANITIZER_INTERCEPT_CFREE \ + (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA) #define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_WCSCAT SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX +#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA) +#define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID #endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc new file mode 100644 index 000000000000..108e196e7a05 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc @@ -0,0 +1,363 @@ +//===-- sanitizer_platform_limits_netbsd.cc -------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific NetBSD data structures. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_NETBSD +#include <arpa/inet.h> +#include <dirent.h> +#include <glob.h> +#include <grp.h> +#include <ifaddrs.h> +#include <limits.h> +#include <link_elf.h> +#include <net/if.h> +#include <net/if_ether.h> +#include <net/ppp_defs.h> +#include <net/route.h> +#include <netdb.h> +#include <netinet/in.h> +#include <netinet/ip_mroute.h> +#include <poll.h> +#include <pthread.h> +#include <pwd.h> +#include <semaphore.h> +#include <signal.h> +#include <stddef.h> +#include <sys/filio.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/mount.h> +#include <sys/mqueue.h> +#include <sys/msg.h> +#include <sys/mtio.h> +#include <sys/ptrace.h> +#include <sys/resource.h> +#include <sys/shm.h> +#include <sys/signal.h> +#include <sys/socket.h> +#include <sys/sockio.h> +#include <sys/soundcard.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/time.h> +#include <sys/timeb.h> +#include <sys/times.h> +#include <sys/timespec.h> +#include <sys/timex.h> +#include <sys/types.h> +#include <sys/ucontext.h> +#include <sys/utsname.h> +#include <term.h> +#include <termios.h> +#include <time.h> +#include <utime.h> +#include <utmp.h> +#include <utmpx.h> +#include <wchar.h> +#include <wordexp.h> + +// Include these after system headers to avoid name clashes and ambiguities. +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_netbsd.h" + +namespace __sanitizer { +unsigned struct_utsname_sz = sizeof(struct utsname); +unsigned struct_stat_sz = sizeof(struct stat); +unsigned struct_rusage_sz = sizeof(struct rusage); +unsigned struct_tm_sz = sizeof(struct tm); +unsigned struct_passwd_sz = sizeof(struct passwd); +unsigned struct_group_sz = sizeof(struct group); +unsigned siginfo_t_sz = sizeof(siginfo_t); +unsigned struct_sigaction_sz = sizeof(struct sigaction); +unsigned struct_itimerval_sz = sizeof(struct itimerval); +unsigned pthread_t_sz = sizeof(pthread_t); +unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); +unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); +unsigned pid_t_sz = sizeof(pid_t); +unsigned timeval_sz = sizeof(timeval); +unsigned uid_t_sz = sizeof(uid_t); +unsigned gid_t_sz = sizeof(gid_t); +unsigned mbstate_t_sz = sizeof(mbstate_t); +unsigned sigset_t_sz = sizeof(sigset_t); +unsigned struct_timezone_sz = sizeof(struct timezone); +unsigned struct_tms_sz = sizeof(struct tms); +unsigned struct_sigevent_sz = sizeof(struct sigevent); +unsigned struct_sched_param_sz = sizeof(struct sched_param); +unsigned struct_sockaddr_sz = sizeof(struct sockaddr); +unsigned ucontext_t_sz = sizeof(ucontext_t); +unsigned struct_rlimit_sz = sizeof(struct rlimit); +unsigned struct_timespec_sz = sizeof(struct timespec); +unsigned struct_utimbuf_sz = sizeof(struct utimbuf); +unsigned struct_itimerspec_sz = sizeof(struct itimerspec); +unsigned struct_timex_sz = sizeof(struct timex); +unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); +unsigned struct_mq_attr_sz = sizeof(struct mq_attr); +unsigned struct_statvfs_sz = sizeof(struct statvfs); + +const uptr sig_ign = (uptr)SIG_IGN; +const uptr sig_dfl = (uptr)SIG_DFL; +const uptr sig_err = (uptr)SIG_ERR; +const uptr sa_siginfo = (uptr)SA_SIGINFO; + +int shmctl_ipc_stat = (int)IPC_STAT; + +unsigned struct_utmp_sz = sizeof(struct utmp); +unsigned struct_utmpx_sz = sizeof(struct utmpx); + +int map_fixed = MAP_FIXED; + +int af_inet = (int)AF_INET; +int af_inet6 = (int)AF_INET6; + +uptr __sanitizer_in_addr_sz(int af) { + if (af == AF_INET) + return sizeof(struct in_addr); + else if (af == AF_INET6) + return sizeof(struct in6_addr); + else + return 0; +} + +unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); + +int glob_nomatch = GLOB_NOMATCH; +int glob_altdirfunc = GLOB_ALTDIRFUNC; + +unsigned path_max = PATH_MAX; + +// ioctl arguments +unsigned struct_ifreq_sz = sizeof(struct ifreq); +unsigned struct_termios_sz = sizeof(struct termios); +unsigned struct_winsize_sz = sizeof(struct winsize); +unsigned struct_mtget_sz = sizeof(struct mtget); +unsigned struct_mtop_sz = sizeof(struct mtop); +unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); +unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); +unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); +unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); + +const unsigned IOCTL_NOT_PRESENT = 0; + +unsigned IOCTL_FIOASYNC = FIOASYNC; +unsigned IOCTL_FIOCLEX = FIOCLEX; +unsigned IOCTL_FIOGETOWN = FIOGETOWN; +unsigned IOCTL_FIONBIO = FIONBIO; +unsigned IOCTL_FIONCLEX = FIONCLEX; +unsigned IOCTL_FIOSETOWN = FIOSETOWN; +unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI; +unsigned IOCTL_SIOCATMARK = SIOCATMARK; +unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI; +unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR; +unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR; +unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF; +unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR; +unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS; +unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC; +unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU; +unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK; +unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; +unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; +unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR; +unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR; +unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS; +unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC; +unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU; +unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK; +unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; +unsigned IOCTL_TIOCCONS = TIOCCONS; +unsigned IOCTL_TIOCEXCL = TIOCEXCL; +unsigned IOCTL_TIOCGETD = TIOCGETD; +unsigned IOCTL_TIOCGPGRP = TIOCGPGRP; +unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ; +unsigned IOCTL_TIOCMBIC = TIOCMBIC; +unsigned IOCTL_TIOCMBIS = TIOCMBIS; +unsigned IOCTL_TIOCMGET = TIOCMGET; +unsigned IOCTL_TIOCMSET = TIOCMSET; +unsigned IOCTL_TIOCNOTTY = TIOCNOTTY; +unsigned IOCTL_TIOCNXCL = TIOCNXCL; +unsigned IOCTL_TIOCOUTQ = TIOCOUTQ; +unsigned IOCTL_TIOCPKT = TIOCPKT; +unsigned IOCTL_TIOCSCTTY = TIOCSCTTY; +unsigned IOCTL_TIOCSETD = TIOCSETD; +unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; +unsigned IOCTL_TIOCSTI = TIOCSTI; +unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; +unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT; +unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT; + +const int si_SEGV_MAPERR = SEGV_MAPERR; +const int si_SEGV_ACCERR = SEGV_ACCERR; +} // namespace __sanitizer + +using namespace __sanitizer; + +COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); + +COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned)); +CHECK_TYPE_SIZE(pthread_key_t); + +// There are more undocumented fields in dl_phdr_info that we are not interested +// in. +COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info)); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum); + +CHECK_TYPE_SIZE(glob_t); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv); +CHECK_SIZE_AND_OFFSET(glob_t, gl_offs); +CHECK_SIZE_AND_OFFSET(glob_t, gl_flags); +CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat); +CHECK_SIZE_AND_OFFSET(glob_t, gl_stat); + +CHECK_TYPE_SIZE(addrinfo); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_family); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr); + +CHECK_TYPE_SIZE(hostent); +CHECK_SIZE_AND_OFFSET(hostent, h_name); +CHECK_SIZE_AND_OFFSET(hostent, h_aliases); +CHECK_SIZE_AND_OFFSET(hostent, h_addrtype); +CHECK_SIZE_AND_OFFSET(hostent, h_length); +CHECK_SIZE_AND_OFFSET(hostent, h_addr_list); + +CHECK_TYPE_SIZE(iovec); +CHECK_SIZE_AND_OFFSET(iovec, iov_base); +CHECK_SIZE_AND_OFFSET(iovec, iov_len); + +CHECK_TYPE_SIZE(msghdr); +CHECK_SIZE_AND_OFFSET(msghdr, msg_name); +CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iov); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_control); +CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_flags); + +CHECK_TYPE_SIZE(cmsghdr); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type); + +COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); +CHECK_SIZE_AND_OFFSET(dirent, d_fileno); +CHECK_SIZE_AND_OFFSET(dirent, d_reclen); + +CHECK_TYPE_SIZE(ifconf); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_len); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu); + +CHECK_TYPE_SIZE(pollfd); +CHECK_SIZE_AND_OFFSET(pollfd, fd); +CHECK_SIZE_AND_OFFSET(pollfd, events); +CHECK_SIZE_AND_OFFSET(pollfd, revents); + +CHECK_TYPE_SIZE(nfds_t); + +CHECK_TYPE_SIZE(sigset_t); + +COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction)); +// Can't write checks for sa_handler and sa_sigaction due to them being +// preprocessor macros. +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); + +CHECK_TYPE_SIZE(wordexp_t); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); + +CHECK_TYPE_SIZE(tm); +CHECK_SIZE_AND_OFFSET(tm, tm_sec); +CHECK_SIZE_AND_OFFSET(tm, tm_min); +CHECK_SIZE_AND_OFFSET(tm, tm_hour); +CHECK_SIZE_AND_OFFSET(tm, tm_mday); +CHECK_SIZE_AND_OFFSET(tm, tm_mon); +CHECK_SIZE_AND_OFFSET(tm, tm_year); +CHECK_SIZE_AND_OFFSET(tm, tm_wday); +CHECK_SIZE_AND_OFFSET(tm, tm_yday); +CHECK_SIZE_AND_OFFSET(tm, tm_isdst); +CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff); +CHECK_SIZE_AND_OFFSET(tm, tm_zone); + +CHECK_TYPE_SIZE(ether_addr); + +CHECK_TYPE_SIZE(ipc_perm); +CHECK_SIZE_AND_OFFSET(ipc_perm, _key); +CHECK_SIZE_AND_OFFSET(ipc_perm, _seq); +CHECK_SIZE_AND_OFFSET(ipc_perm, uid); +CHECK_SIZE_AND_OFFSET(ipc_perm, gid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); +CHECK_SIZE_AND_OFFSET(ipc_perm, mode); + +CHECK_TYPE_SIZE(shmid_ds); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch); + +CHECK_TYPE_SIZE(clock_t); + +CHECK_TYPE_SIZE(ifaddrs); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask); +// Compare against the union, because we can't reach into the union in a +// compliant way. +#ifdef ifa_dstaddr +#undef ifa_dstaddr +#endif +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); + +CHECK_TYPE_SIZE(timeb); +CHECK_SIZE_AND_OFFSET(timeb, time); +CHECK_SIZE_AND_OFFSET(timeb, millitm); +CHECK_SIZE_AND_OFFSET(timeb, timezone); +CHECK_SIZE_AND_OFFSET(timeb, dstflag); + +CHECK_TYPE_SIZE(passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_name); +CHECK_SIZE_AND_OFFSET(passwd, pw_passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_uid); +CHECK_SIZE_AND_OFFSET(passwd, pw_gid); +CHECK_SIZE_AND_OFFSET(passwd, pw_dir); +CHECK_SIZE_AND_OFFSET(passwd, pw_shell); + +CHECK_SIZE_AND_OFFSET(passwd, pw_gecos); + +CHECK_TYPE_SIZE(group); +CHECK_SIZE_AND_OFFSET(group, gr_name); +CHECK_SIZE_AND_OFFSET(group, gr_passwd); +CHECK_SIZE_AND_OFFSET(group, gr_gid); +CHECK_SIZE_AND_OFFSET(group, gr_mem); + +#endif // SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h new file mode 100644 index 000000000000..6823004d7b41 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -0,0 +1,582 @@ +//===-- sanitizer_platform_limits_netbsd.h --------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific NetBSD data structures. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_PLATFORM_LIMITS_NETBSD_H +#define SANITIZER_PLATFORM_LIMITS_NETBSD_H + +#if SANITIZER_NETBSD + +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" + +#define _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, shift) \ + ((link_map *)((handle) == nullptr ? nullptr : ((char *)(handle) + (shift)))) + +#if defined(__x86_64__) +#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 312) +#elif defined(__i386__) +#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 164) +#endif + +namespace __sanitizer { +extern unsigned struct_utsname_sz; +extern unsigned struct_stat_sz; +extern unsigned struct_rusage_sz; +extern unsigned siginfo_t_sz; +extern unsigned struct_itimerval_sz; +extern unsigned pthread_t_sz; +extern unsigned pthread_mutex_t_sz; +extern unsigned pthread_cond_t_sz; +extern unsigned pid_t_sz; +extern unsigned timeval_sz; +extern unsigned uid_t_sz; +extern unsigned gid_t_sz; +extern unsigned mbstate_t_sz; +extern unsigned struct_timezone_sz; +extern unsigned struct_tms_sz; +extern unsigned struct_itimerspec_sz; +extern unsigned struct_sigevent_sz; +extern unsigned struct_sched_param_sz; +extern unsigned struct_statfs_sz; +extern unsigned struct_sockaddr_sz; +extern unsigned ucontext_t_sz; + +extern unsigned struct_rlimit_sz; +extern unsigned struct_utimbuf_sz; +extern unsigned struct_timespec_sz; + +struct __sanitizer_iocb { + u64 aio_offset; + uptr aio_buf; + long aio_nbytes; + u32 aio_fildes; + u32 aio_lio_opcode; + long aio_reqprio; +#if SANITIZER_WORDSIZE == 64 + u8 aio_sigevent[32]; +#else + u8 aio_sigevent[20]; +#endif + u32 _state; + u32 _errno; + long _retval; +}; + +struct __sanitizer___sysctl_args { + int *name; + int nlen; + void *oldval; + uptr *oldlenp; + void *newval; + uptr newlen; +}; + +struct __sanitizer_sem_t { + uptr data[5]; +}; + +struct __sanitizer_ipc_perm { + u32 uid; + u32 gid; + u32 cuid; + u32 cgid; + u32 mode; + unsigned short _seq; + long _key; +}; + +struct __sanitizer_shmid_ds { + __sanitizer_ipc_perm shm_perm; + unsigned long shm_segsz; + u32 shm_lpid; + u32 shm_cpid; + unsigned int shm_nattch; + u64 shm_atime; + u64 shm_dtime; + u64 shm_ctime; + void *_shm_internal; +}; + +extern unsigned struct_msqid_ds_sz; +extern unsigned struct_mq_attr_sz; +extern unsigned struct_timex_sz; +extern unsigned struct_statvfs_sz; + +struct __sanitizer_iovec { + void *iov_base; + uptr iov_len; +}; + +struct __sanitizer_ifaddrs { + struct __sanitizer_ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + void *ifa_addr; // (struct sockaddr *) + void *ifa_netmask; // (struct sockaddr *) + void *ifa_dstaddr; // (struct sockaddr *) + void *ifa_data; + unsigned int ifa_addrflags; +}; + +typedef unsigned __sanitizer_pthread_key_t; + +typedef long long __sanitizer_time_t; + +struct __sanitizer_passwd { + char *pw_name; + char *pw_passwd; + int pw_uid; + int pw_gid; + __sanitizer_time_t pw_change; + char *pw_class; + char *pw_gecos; + char *pw_dir; + char *pw_shell; + __sanitizer_time_t pw_expire; +}; + +struct __sanitizer_group { + char *gr_name; + char *gr_passwd; + int gr_gid; + char **gr_mem; +}; + +struct __sanitizer_timeb { + __sanitizer_time_t time; + unsigned short millitm; + short timezone; + short dstflag; +}; + +struct __sanitizer_ether_addr { + u8 octet[6]; +}; + +struct __sanitizer_tm { + int tm_sec; + int tm_min; + int tm_hour; + int tm_mday; + int tm_mon; + int tm_year; + int tm_wday; + int tm_yday; + int tm_isdst; + long int tm_gmtoff; + const char *tm_zone; +}; + +struct __sanitizer_msghdr { + void *msg_name; + unsigned msg_namelen; + struct __sanitizer_iovec *msg_iov; + unsigned msg_iovlen; + void *msg_control; + unsigned msg_controllen; + int msg_flags; +}; +struct __sanitizer_cmsghdr { + unsigned cmsg_len; + int cmsg_level; + int cmsg_type; +}; + +struct __sanitizer_dirent { + u64 d_fileno; + u16 d_reclen; + // more fields that we don't care about +}; + +typedef int __sanitizer_clock_t; +typedef int __sanitizer_clockid_t; + +typedef u32 __sanitizer___kernel_uid_t; +typedef u32 __sanitizer___kernel_gid_t; +typedef u64 __sanitizer___kernel_off_t; +typedef struct { + u32 fds_bits[8]; +} __sanitizer___kernel_fd_set; + +typedef struct { + unsigned int pta_magic; + int pta_flags; + void *pta_private; +} __sanitizer_pthread_attr_t; + +struct __sanitizer_sigset_t { + // uint32_t * 4 + unsigned int __bits[4]; +}; + +struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; +}; + +using __sanitizer_sighandler_ptr = void (*)(int sig); +using __sanitizer_sigactionhandler_ptr = void (*)(int sig, + __sanitizer_siginfo *siginfo, + void *uctx); + +struct __sanitizer_sigaction { + union { + __sanitizer_sighandler_ptr handler; + __sanitizer_sigactionhandler_ptr sigaction; + }; + __sanitizer_sigset_t sa_mask; + int sa_flags; +}; + +typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; + +struct __sanitizer_kernel_sigaction_t { + union { + void (*handler)(int signo); + void (*sigaction)(int signo, void *info, void *ctx); + }; + unsigned long sa_flags; + void (*sa_restorer)(void); + __sanitizer_kernel_sigset_t sa_mask; +}; + +extern const uptr sig_ign; +extern const uptr sig_dfl; +extern const uptr sig_err; +extern const uptr sa_siginfo; + +extern int af_inet; +extern int af_inet6; +uptr __sanitizer_in_addr_sz(int af); + +struct __sanitizer_dl_phdr_info { + uptr dlpi_addr; + const char *dlpi_name; + const void *dlpi_phdr; + short dlpi_phnum; +}; + +extern unsigned struct_ElfW_Phdr_sz; + +struct __sanitizer_addrinfo { + int ai_flags; + int ai_family; + int ai_socktype; + int ai_protocol; + unsigned ai_addrlen; + char *ai_canonname; + void *ai_addr; + struct __sanitizer_addrinfo *ai_next; +}; + +struct __sanitizer_hostent { + char *h_name; + char **h_aliases; + int h_addrtype; + int h_length; + char **h_addr_list; +}; + +struct __sanitizer_pollfd { + int fd; + short events; + short revents; +}; + +typedef unsigned __sanitizer_nfds_t; + +struct __sanitizer_glob_t { + uptr gl_pathc; + uptr gl_matchc; + uptr gl_offs; + int gl_flags; + char **gl_pathv; + int (*gl_errfunc)(const char *, int); + void (*gl_closedir)(void *dirp); + struct dirent *(*gl_readdir)(void *dirp); + void *(*gl_opendir)(const char *); + int (*gl_lstat)(const char *, void * /* struct stat* */); + int (*gl_stat)(const char *, void * /* struct stat* */); +}; + +extern int glob_nomatch; +extern int glob_altdirfunc; + +extern unsigned path_max; + +struct __sanitizer_wordexp_t { + uptr we_wordc; + char **we_wordv; + uptr we_offs; + char *we_strings; + uptr we_nbytes; +}; + +typedef char __sanitizer_FILE; +#define SANITIZER_HAS_STRUCT_FILE 0 + +extern int shmctl_ipc_stat; + +// This simplifies generic code +#define struct_shminfo_sz -1 +#define struct_shm_info_sz -1 +#define shmctl_shm_stat -1 +#define shmctl_ipc_info -1 +#define shmctl_shm_info -1 + +extern unsigned struct_utmp_sz; +extern unsigned struct_utmpx_sz; + +extern int map_fixed; + +// ioctl arguments +struct __sanitizer_ifconf { + int ifc_len; + union { + void *ifcu_req; + } ifc_ifcu; +}; + +#define IOC_NRBITS 8 +#define IOC_TYPEBITS 8 +#define IOC_SIZEBITS 14 +#define IOC_DIRBITS 2 +#define IOC_NONE 0U +#define IOC_WRITE 1U +#define IOC_READ 2U +#define IOC_NRMASK ((1 << IOC_NRBITS) - 1) +#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1) +#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1) +#undef IOC_DIRMASK +#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1) +#define IOC_NRSHIFT 0 +#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS) +#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS) +#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS) +#define EVIOC_EV_MAX 0x1f +#define EVIOC_ABS_MAX 0x3f + +#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) +#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) +#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) +#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) + +extern unsigned struct_ifreq_sz; +extern unsigned struct_termios_sz; +extern unsigned struct_winsize_sz; + +extern unsigned struct_arpreq_sz; + +extern unsigned struct_mtget_sz; +extern unsigned struct_mtop_sz; +extern unsigned struct_rtentry_sz; +extern unsigned struct_sbi_instrument_sz; +extern unsigned struct_seq_event_rec_sz; +extern unsigned struct_synth_info_sz; +extern unsigned struct_vt_mode_sz; +extern unsigned struct_audio_buf_info_sz; +extern unsigned struct_ppp_stats_sz; +extern unsigned struct_sioc_sg_req_sz; +extern unsigned struct_sioc_vif_req_sz; + +// ioctl request identifiers + +// A special value to mark ioctls that are not present on the target platform, +// when it can not be determined without including any system headers. +extern const unsigned IOCTL_NOT_PRESENT; + +extern unsigned IOCTL_FIOASYNC; +extern unsigned IOCTL_FIOCLEX; +extern unsigned IOCTL_FIOGETOWN; +extern unsigned IOCTL_FIONBIO; +extern unsigned IOCTL_FIONCLEX; +extern unsigned IOCTL_FIOSETOWN; +extern unsigned IOCTL_SIOCADDMULTI; +extern unsigned IOCTL_SIOCATMARK; +extern unsigned IOCTL_SIOCDELMULTI; +extern unsigned IOCTL_SIOCGIFADDR; +extern unsigned IOCTL_SIOCGIFBRDADDR; +extern unsigned IOCTL_SIOCGIFCONF; +extern unsigned IOCTL_SIOCGIFDSTADDR; +extern unsigned IOCTL_SIOCGIFFLAGS; +extern unsigned IOCTL_SIOCGIFMETRIC; +extern unsigned IOCTL_SIOCGIFMTU; +extern unsigned IOCTL_SIOCGIFNETMASK; +extern unsigned IOCTL_SIOCGPGRP; +extern unsigned IOCTL_SIOCSIFADDR; +extern unsigned IOCTL_SIOCSIFBRDADDR; +extern unsigned IOCTL_SIOCSIFDSTADDR; +extern unsigned IOCTL_SIOCSIFFLAGS; +extern unsigned IOCTL_SIOCSIFMETRIC; +extern unsigned IOCTL_SIOCSIFMTU; +extern unsigned IOCTL_SIOCSIFNETMASK; +extern unsigned IOCTL_SIOCSPGRP; +extern unsigned IOCTL_TIOCCONS; +extern unsigned IOCTL_TIOCEXCL; +extern unsigned IOCTL_TIOCGETD; +extern unsigned IOCTL_TIOCGPGRP; +extern unsigned IOCTL_TIOCGWINSZ; +extern unsigned IOCTL_TIOCMBIC; +extern unsigned IOCTL_TIOCMBIS; +extern unsigned IOCTL_TIOCMGET; +extern unsigned IOCTL_TIOCMSET; +extern unsigned IOCTL_TIOCNOTTY; +extern unsigned IOCTL_TIOCNXCL; +extern unsigned IOCTL_TIOCOUTQ; +extern unsigned IOCTL_TIOCPKT; +extern unsigned IOCTL_TIOCSCTTY; +extern unsigned IOCTL_TIOCSETD; +extern unsigned IOCTL_TIOCSPGRP; +extern unsigned IOCTL_TIOCSTI; +extern unsigned IOCTL_TIOCSWINSZ; +extern unsigned IOCTL_SIOCGETSGCNT; +extern unsigned IOCTL_SIOCGETVIFCNT; +extern unsigned IOCTL_MTIOCGET; +extern unsigned IOCTL_MTIOCTOP; +extern unsigned IOCTL_SIOCADDRT; +extern unsigned IOCTL_SIOCDELRT; +extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE; +extern unsigned IOCTL_SNDCTL_DSP_GETFMTS; +extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK; +extern unsigned IOCTL_SNDCTL_DSP_POST; +extern unsigned IOCTL_SNDCTL_DSP_RESET; +extern unsigned IOCTL_SNDCTL_DSP_SETFMT; +extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT; +extern unsigned IOCTL_SNDCTL_DSP_SPEED; +extern unsigned IOCTL_SNDCTL_DSP_STEREO; +extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE; +extern unsigned IOCTL_SNDCTL_DSP_SYNC; +extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE; +extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR; +extern unsigned IOCTL_SNDCTL_MIDI_INFO; +extern unsigned IOCTL_SNDCTL_MIDI_PRETIME; +extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE; +extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT; +extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT; +extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS; +extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS; +extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND; +extern unsigned IOCTL_SNDCTL_SEQ_PANIC; +extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE; +extern unsigned IOCTL_SNDCTL_SEQ_RESET; +extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES; +extern unsigned IOCTL_SNDCTL_SEQ_SYNC; +extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI; +extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD; +extern unsigned IOCTL_SNDCTL_SYNTH_INFO; +extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL; +extern unsigned IOCTL_SNDCTL_TMR_CONTINUE; +extern unsigned IOCTL_SNDCTL_TMR_METRONOME; +extern unsigned IOCTL_SNDCTL_TMR_SELECT; +extern unsigned IOCTL_SNDCTL_TMR_SOURCE; +extern unsigned IOCTL_SNDCTL_TMR_START; +extern unsigned IOCTL_SNDCTL_TMR_STOP; +extern unsigned IOCTL_SNDCTL_TMR_TEMPO; +extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE; +extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM; +extern unsigned IOCTL_SOUND_MIXER_READ_BASS; +extern unsigned IOCTL_SOUND_MIXER_READ_CAPS; +extern unsigned IOCTL_SOUND_MIXER_READ_CD; +extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK; +extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE; +extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN; +extern unsigned IOCTL_SOUND_MIXER_READ_IMIX; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE1; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE2; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE3; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE; +extern unsigned IOCTL_SOUND_MIXER_READ_LOUD; +extern unsigned IOCTL_SOUND_MIXER_READ_MIC; +extern unsigned IOCTL_SOUND_MIXER_READ_MUTE; +extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN; +extern unsigned IOCTL_SOUND_MIXER_READ_PCM; +extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV; +extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK; +extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC; +extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER; +extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS; +extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH; +extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE; +extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME; +extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM; +extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS; +extern unsigned IOCTL_SOUND_MIXER_WRITE_CD; +extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN; +extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD; +extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC; +extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN; +extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM; +extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV; +extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC; +extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER; +extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH; +extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME; +extern unsigned IOCTL_SOUND_PCM_READ_BITS; +extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS; +extern unsigned IOCTL_SOUND_PCM_READ_FILTER; +extern unsigned IOCTL_SOUND_PCM_READ_RATE; +extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS; +extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER; +extern unsigned IOCTL_VT_ACTIVATE; +extern unsigned IOCTL_VT_GETMODE; +extern unsigned IOCTL_VT_OPENQRY; +extern unsigned IOCTL_VT_RELDISP; +extern unsigned IOCTL_VT_SETMODE; +extern unsigned IOCTL_VT_WAITACTIVE; +extern unsigned IOCTL_KDDISABIO; +extern unsigned IOCTL_KDENABIO; +extern unsigned IOCTL_KDGETLED; +extern unsigned IOCTL_KDGKBMODE; +extern unsigned IOCTL_KDGKBTYPE; +extern unsigned IOCTL_KDMKTONE; +extern unsigned IOCTL_KDSETLED; +extern unsigned IOCTL_KDSETMODE; +extern unsigned IOCTL_KDSKBMODE; + +extern const int si_SEGV_MAPERR; +extern const int si_SEGV_ACCERR; +} // namespace __sanitizer + +#define CHECK_TYPE_SIZE(TYPE) \ + COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE)) + +#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \ + sizeof(((CLASS *)NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \ + offsetof(CLASS, MEMBER)) + +// For sigaction, which is a function and struct at the same time, +// and thus requires explicit "struct" in sizeof() expression. +#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \ + sizeof(((struct CLASS *)NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ + offsetof(struct CLASS, MEMBER)) + +#define SIGACTION_SYMNAME __sigaction14 + +#endif // SANITIZER_NETBSD + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index 83f4fd22f623..f12e8206abe6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -210,6 +210,7 @@ namespace __sanitizer { unsigned struct_sigaction_sz = sizeof(struct sigaction); unsigned struct_itimerval_sz = sizeof(struct itimerval); unsigned pthread_t_sz = sizeof(pthread_t); + unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); unsigned pid_t_sz = sizeof(pid_t); unsigned timeval_sz = sizeof(timeval); @@ -264,9 +265,10 @@ namespace __sanitizer { unsigned struct_statvfs_sz = sizeof(struct statvfs); #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID - uptr sig_ign = (uptr)SIG_IGN; - uptr sig_dfl = (uptr)SIG_DFL; - uptr sa_siginfo = (uptr)SA_SIGINFO; + const uptr sig_ign = (uptr)SIG_IGN; + const uptr sig_dfl = (uptr)SIG_DFL; + const uptr sig_err = (uptr)SIG_ERR; + const uptr sa_siginfo = (uptr)SA_SIGINFO; #if SANITIZER_LINUX int e_tabsz = (int)E_TABSZ; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 808418012d75..2853a84d43ae 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -15,6 +15,8 @@ #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H #define SANITIZER_PLATFORM_LIMITS_POSIX_H +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC + #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" @@ -44,6 +46,7 @@ namespace __sanitizer { extern unsigned siginfo_t_sz; extern unsigned struct_itimerval_sz; extern unsigned pthread_t_sz; + extern unsigned pthread_mutex_t_sz; extern unsigned pthread_cond_t_sz; extern unsigned pid_t_sz; extern unsigned timeval_sz; @@ -535,7 +538,7 @@ namespace __sanitizer { typedef long __sanitizer_clock_t; #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD typedef int __sanitizer_clockid_t; #endif @@ -596,13 +599,22 @@ namespace __sanitizer { }; #endif + struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; + }; + + using __sanitizer_sighandler_ptr = void (*)(int sig); + using __sanitizer_sigactionhandler_ptr = + void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx); + // Linux system headers define the 'sa_handler' and 'sa_sigaction' macros. #if SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 64) struct __sanitizer_sigaction { unsigned sa_flags; union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; __sanitizer_sigset_t sa_mask; void (*sa_restorer)(); @@ -611,16 +623,16 @@ namespace __sanitizer { struct __sanitizer_sigaction { unsigned sa_flags; union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; __sanitizer_sigset_t sa_mask; }; #elif SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32) struct __sanitizer_sigaction { union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; __sanitizer_sigset_t sa_mask; uptr sa_flags; @@ -632,8 +644,8 @@ namespace __sanitizer { unsigned int sa_flags; #endif union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; #if SANITIZER_FREEBSD int sa_flags; @@ -692,7 +704,7 @@ namespace __sanitizer { unsigned int sa_flags; union { void (*handler)(int signo); - void (*sigaction)(int signo, void *info, void *ctx); + void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx); }; __sanitizer_kernel_sigset_t sa_mask; void (*sa_restorer)(void); @@ -701,7 +713,7 @@ namespace __sanitizer { struct __sanitizer_kernel_sigaction_t { union { void (*handler)(int signo); - void (*sigaction)(int signo, void *info, void *ctx); + void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx); }; unsigned long sa_flags; void (*sa_restorer)(void); @@ -709,9 +721,10 @@ namespace __sanitizer { }; #endif - extern uptr sig_ign; - extern uptr sig_dfl; - extern uptr sa_siginfo; + extern const uptr sig_ign; + extern const uptr sig_dfl; + extern const uptr sig_err; + extern const uptr sa_siginfo; #if SANITIZER_LINUX extern int e_tabsz; @@ -1489,4 +1502,8 @@ struct __sanitizer_cookie_io_functions_t { COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ offsetof(struct CLASS, MEMBER)) +#define SIGACTION_SYMNAME sigaction + +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC + #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc new file mode 100644 index 000000000000..15b80a81ae3b --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc @@ -0,0 +1,366 @@ +//===-- sanitizer_platform_limits_solaris.cc ------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific Solaris data structures. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_SOLARIS +#include <arpa/inet.h> +#include <dirent.h> +#include <glob.h> +#include <grp.h> +#include <ifaddrs.h> +#include <limits.h> +#include <link.h> +#include <net/if.h> +#include <net/route.h> +#include <netdb.h> +#include <netinet/ip_mroute.h> +#include <poll.h> +#include <pthread.h> +#include <pwd.h> +#include <rpc/xdr.h> +#include <semaphore.h> +#include <signal.h> +#include <stddef.h> +#include <sys/ethernet.h> +#include <sys/filio.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/mount.h> +#include <sys/mtio.h> +#include <sys/ptyvar.h> +#include <sys/resource.h> +#include <sys/shm.h> +#include <sys/socket.h> +#include <sys/sockio.h> +#include <sys/stat.h> +#include <sys/statfs.h> +#include <sys/statvfs.h> +#include <sys/time.h> +#include <sys/timeb.h> +#include <sys/times.h> +#include <sys/types.h> +#include <sys/utsname.h> +#include <termios.h> +#include <time.h> +#include <utmp.h> +#include <utmpx.h> +#include <wchar.h> +#include <wordexp.h> + +// Include these after system headers to avoid name clashes and ambiguities. +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_solaris.h" + +namespace __sanitizer { + unsigned struct_utsname_sz = sizeof(struct utsname); + unsigned struct_stat_sz = sizeof(struct stat); + unsigned struct_stat64_sz = sizeof(struct stat64); + unsigned struct_rusage_sz = sizeof(struct rusage); + unsigned struct_tm_sz = sizeof(struct tm); + unsigned struct_passwd_sz = sizeof(struct passwd); + unsigned struct_group_sz = sizeof(struct group); + unsigned siginfo_t_sz = sizeof(siginfo_t); + unsigned struct_sigaction_sz = sizeof(struct sigaction); + unsigned struct_itimerval_sz = sizeof(struct itimerval); + unsigned pthread_t_sz = sizeof(pthread_t); + unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); + unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); + unsigned pid_t_sz = sizeof(pid_t); + unsigned timeval_sz = sizeof(timeval); + unsigned uid_t_sz = sizeof(uid_t); + unsigned gid_t_sz = sizeof(gid_t); + unsigned mbstate_t_sz = sizeof(mbstate_t); + unsigned sigset_t_sz = sizeof(sigset_t); + unsigned struct_timezone_sz = sizeof(struct timezone); + unsigned struct_tms_sz = sizeof(struct tms); + unsigned struct_sigevent_sz = sizeof(struct sigevent); + unsigned struct_sched_param_sz = sizeof(struct sched_param); + unsigned struct_statfs_sz = sizeof(struct statfs); + unsigned struct_sockaddr_sz = sizeof(struct sockaddr); + unsigned ucontext_t_sz = sizeof(ucontext_t); + unsigned struct_timespec_sz = sizeof(struct timespec); +#if SANITIZER_SOLARIS32 + unsigned struct_statvfs64_sz = sizeof(struct statvfs64); +#endif + unsigned struct_statvfs_sz = sizeof(struct statvfs); + + const uptr sig_ign = (uptr)SIG_IGN; + const uptr sig_dfl = (uptr)SIG_DFL; + const uptr sig_err = (uptr)SIG_ERR; + const uptr sa_siginfo = (uptr)SA_SIGINFO; + + int shmctl_ipc_stat = (int)IPC_STAT; + + unsigned struct_utmp_sz = sizeof(struct utmp); + unsigned struct_utmpx_sz = sizeof(struct utmpx); + + int map_fixed = MAP_FIXED; + + int af_inet = (int)AF_INET; + int af_inet6 = (int)AF_INET6; + + uptr __sanitizer_in_addr_sz(int af) { + if (af == AF_INET) + return sizeof(struct in_addr); + else if (af == AF_INET6) + return sizeof(struct in6_addr); + else + return 0; + } + + unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr)); + + int glob_nomatch = GLOB_NOMATCH; + + unsigned path_max = PATH_MAX; + + // ioctl arguments + unsigned struct_ifreq_sz = sizeof(struct ifreq); + unsigned struct_termios_sz = sizeof(struct termios); + unsigned struct_winsize_sz = sizeof(struct winsize); + + unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); + unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); + + const unsigned IOCTL_NOT_PRESENT = 0; + + unsigned IOCTL_FIOASYNC = FIOASYNC; + unsigned IOCTL_FIOCLEX = FIOCLEX; + unsigned IOCTL_FIOGETOWN = FIOGETOWN; + unsigned IOCTL_FIONBIO = FIONBIO; + unsigned IOCTL_FIONCLEX = FIONCLEX; + unsigned IOCTL_FIOSETOWN = FIOSETOWN; + unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI; + unsigned IOCTL_SIOCATMARK = SIOCATMARK; + unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI; + unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR; + unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR; + unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF; + unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR; + unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS; + unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC; + unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU; + unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK; + unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; + unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; + unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR; + unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR; + unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS; + unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC; + unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU; + unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK; + unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; + unsigned IOCTL_TIOCEXCL = TIOCEXCL; + unsigned IOCTL_TIOCGETD = TIOCGETD; + unsigned IOCTL_TIOCGPGRP = TIOCGPGRP; + unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ; + unsigned IOCTL_TIOCMBIC = TIOCMBIC; + unsigned IOCTL_TIOCMBIS = TIOCMBIS; + unsigned IOCTL_TIOCMGET = TIOCMGET; + unsigned IOCTL_TIOCMSET = TIOCMSET; + unsigned IOCTL_TIOCNOTTY = TIOCNOTTY; + unsigned IOCTL_TIOCNXCL = TIOCNXCL; + unsigned IOCTL_TIOCOUTQ = TIOCOUTQ; + unsigned IOCTL_TIOCPKT = TIOCPKT; + unsigned IOCTL_TIOCSCTTY = TIOCSCTTY; + unsigned IOCTL_TIOCSETD = TIOCSETD; + unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; + unsigned IOCTL_TIOCSTI = TIOCSTI; + unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; + + unsigned IOCTL_MTIOCGET = MTIOCGET; + unsigned IOCTL_MTIOCTOP = MTIOCTOP; + + const int si_SEGV_MAPERR = SEGV_MAPERR; + const int si_SEGV_ACCERR = SEGV_ACCERR; +} // namespace __sanitizer + +using namespace __sanitizer; + +COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); + +COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned)); +CHECK_TYPE_SIZE(pthread_key_t); + +// There are more undocumented fields in dl_phdr_info that we are not interested +// in. +COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info)); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum); + +CHECK_TYPE_SIZE(glob_t); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv); +CHECK_SIZE_AND_OFFSET(glob_t, gl_offs); + +CHECK_TYPE_SIZE(addrinfo); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_family); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr); + +CHECK_TYPE_SIZE(hostent); +CHECK_SIZE_AND_OFFSET(hostent, h_name); +CHECK_SIZE_AND_OFFSET(hostent, h_aliases); +CHECK_SIZE_AND_OFFSET(hostent, h_addrtype); +CHECK_SIZE_AND_OFFSET(hostent, h_length); +CHECK_SIZE_AND_OFFSET(hostent, h_addr_list); + +CHECK_TYPE_SIZE(iovec); +CHECK_SIZE_AND_OFFSET(iovec, iov_base); +CHECK_SIZE_AND_OFFSET(iovec, iov_len); + +CHECK_TYPE_SIZE(msghdr); +CHECK_SIZE_AND_OFFSET(msghdr, msg_name); +CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iov); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_control); +CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_flags); + +CHECK_TYPE_SIZE(cmsghdr); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type); + +COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); +CHECK_SIZE_AND_OFFSET(dirent, d_ino); +CHECK_SIZE_AND_OFFSET(dirent, d_off); +CHECK_SIZE_AND_OFFSET(dirent, d_reclen); + +#if SANITIZER_SOLARIS32 +COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64)); +CHECK_SIZE_AND_OFFSET(dirent64, d_ino); +CHECK_SIZE_AND_OFFSET(dirent64, d_off); +CHECK_SIZE_AND_OFFSET(dirent64, d_reclen); +#endif + +CHECK_TYPE_SIZE(ifconf); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_len); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu); + +CHECK_TYPE_SIZE(pollfd); +CHECK_SIZE_AND_OFFSET(pollfd, fd); +CHECK_SIZE_AND_OFFSET(pollfd, events); +CHECK_SIZE_AND_OFFSET(pollfd, revents); + +CHECK_TYPE_SIZE(nfds_t); + +CHECK_TYPE_SIZE(sigset_t); + +COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction)); +// Can't write checks for sa_handler and sa_sigaction due to them being +// preprocessor macros. +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags); + +CHECK_TYPE_SIZE(wordexp_t); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); + +CHECK_TYPE_SIZE(tm); +CHECK_SIZE_AND_OFFSET(tm, tm_sec); +CHECK_SIZE_AND_OFFSET(tm, tm_min); +CHECK_SIZE_AND_OFFSET(tm, tm_hour); +CHECK_SIZE_AND_OFFSET(tm, tm_mday); +CHECK_SIZE_AND_OFFSET(tm, tm_mon); +CHECK_SIZE_AND_OFFSET(tm, tm_year); +CHECK_SIZE_AND_OFFSET(tm, tm_wday); +CHECK_SIZE_AND_OFFSET(tm, tm_yday); +CHECK_SIZE_AND_OFFSET(tm, tm_isdst); + +CHECK_TYPE_SIZE(ether_addr); + +CHECK_TYPE_SIZE(ipc_perm); +CHECK_SIZE_AND_OFFSET(ipc_perm, key); +CHECK_SIZE_AND_OFFSET(ipc_perm, seq); +CHECK_SIZE_AND_OFFSET(ipc_perm, uid); +CHECK_SIZE_AND_OFFSET(ipc_perm, gid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); +CHECK_SIZE_AND_OFFSET(ipc_perm, mode); + +CHECK_TYPE_SIZE(shmid_ds); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch); + +CHECK_TYPE_SIZE(clock_t); + +CHECK_TYPE_SIZE(ifaddrs); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask); +// Compare against the union, because we can't reach into the union in a +// compliant way. +#ifdef ifa_dstaddr +#undef ifa_dstaddr +#endif +COMPILER_CHECK(sizeof(((__sanitizer_ifaddrs *)nullptr)->ifa_dstaddr) == + sizeof(((ifaddrs *)nullptr)->ifa_ifu)); +COMPILER_CHECK(offsetof(__sanitizer_ifaddrs, ifa_dstaddr) == + offsetof(ifaddrs, ifa_ifu)); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); + +CHECK_TYPE_SIZE(timeb); +CHECK_SIZE_AND_OFFSET(timeb, time); +CHECK_SIZE_AND_OFFSET(timeb, millitm); +CHECK_SIZE_AND_OFFSET(timeb, timezone); +CHECK_SIZE_AND_OFFSET(timeb, dstflag); + +CHECK_TYPE_SIZE(passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_name); +CHECK_SIZE_AND_OFFSET(passwd, pw_passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_uid); +CHECK_SIZE_AND_OFFSET(passwd, pw_gid); +CHECK_SIZE_AND_OFFSET(passwd, pw_dir); +CHECK_SIZE_AND_OFFSET(passwd, pw_shell); + +CHECK_SIZE_AND_OFFSET(passwd, pw_gecos); + +CHECK_TYPE_SIZE(group); +CHECK_SIZE_AND_OFFSET(group, gr_name); +CHECK_SIZE_AND_OFFSET(group, gr_passwd); +CHECK_SIZE_AND_OFFSET(group, gr_gid); +CHECK_SIZE_AND_OFFSET(group, gr_mem); + +CHECK_TYPE_SIZE(XDR); +CHECK_SIZE_AND_OFFSET(XDR, x_op); +CHECK_SIZE_AND_OFFSET(XDR, x_ops); +CHECK_SIZE_AND_OFFSET(XDR, x_public); +CHECK_SIZE_AND_OFFSET(XDR, x_private); +CHECK_SIZE_AND_OFFSET(XDR, x_base); +CHECK_SIZE_AND_OFFSET(XDR, x_handy); +COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE); +COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE); +COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE); + +CHECK_TYPE_SIZE(sem_t); + +#endif // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h new file mode 100644 index 000000000000..a9db71b99c81 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h @@ -0,0 +1,485 @@ +//===-- sanitizer_platform_limits_solaris.h -------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific Solaris data structures. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_PLATFORM_LIMITS_SOLARIS_H +#define SANITIZER_PLATFORM_LIMITS_SOLARIS_H + +#if SANITIZER_SOLARIS + +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" + +namespace __sanitizer { +extern unsigned struct_utsname_sz; +extern unsigned struct_stat_sz; +extern unsigned struct_stat64_sz; +extern unsigned struct_rusage_sz; +extern unsigned siginfo_t_sz; +extern unsigned struct_itimerval_sz; +extern unsigned pthread_t_sz; +extern unsigned pthread_mutex_t_sz; +extern unsigned pthread_cond_t_sz; +extern unsigned pid_t_sz; +extern unsigned timeval_sz; +extern unsigned uid_t_sz; +extern unsigned gid_t_sz; +extern unsigned mbstate_t_sz; +extern unsigned struct_timezone_sz; +extern unsigned struct_tms_sz; +extern unsigned struct_itimerspec_sz; +extern unsigned struct_sigevent_sz; +extern unsigned struct_sched_param_sz; +extern unsigned struct_statfs64_sz; +extern unsigned struct_statfs_sz; +extern unsigned struct_sockaddr_sz; +extern unsigned ucontext_t_sz; + +extern unsigned struct_timespec_sz; +extern unsigned struct_rlimit_sz; +extern unsigned struct_utimbuf_sz; + +struct __sanitizer_sem_t { + //u64 data[6]; + u32 sem_count; + u16 sem_type; + u16 sem_magic; + u64 sem_pad1[3]; + u64 sem_pad2[2]; +}; + +struct __sanitizer_ipc_perm { + unsigned int uid; // uid_t + unsigned int gid; // gid_t + unsigned int cuid; // uid_t + unsigned int cgid; // gid_t + unsigned int mode; // mode_t + unsigned int seq; // uint_t + int key; // key_t +#if !defined(_LP64) + int pad[4]; +#endif + }; + +struct __sanitizer_shmid_ds { + __sanitizer_ipc_perm shm_perm; + unsigned long shm_segsz; // size_t + unsigned long shm_flags; // uintptr_t + unsigned short shm_lkcnt; // ushort_t + int shm_lpid; // pid_t + int shm_cpid; // pid_t + unsigned long shm_nattch; // shmatt_t + unsigned long shm_cnattch; // ulong_t +#if defined(_LP64) + long shm_atime; // time_t + long shm_dtime; + long shm_ctime; + void *shm_amp; + u64 shm_gransize; // uint64_t + u64 shm_allocated; // uint64_t + u64 shm_pad4[1]; // int64_t +#else + long shm_atime; // time_t + int shm_pad1; // int32_t + long shm_dtime; // time_t + int shm_pad2; // int32_t + long shm_ctime; // time_t + void *shm_amp; + u64 shm_gransize; // uint64_t + u64 shm_allocated; // uint64_t +#endif +}; + +extern unsigned struct_statvfs_sz; +#if SANITIZER_SOLARIS32 +extern unsigned struct_statvfs64_sz; +#endif + +struct __sanitizer_iovec { + void *iov_base; + uptr iov_len; +}; + +struct __sanitizer_ifaddrs { + struct __sanitizer_ifaddrs *ifa_next; + char *ifa_name; + u64 ifa_flags; // uint64_t + void *ifa_addr; // (struct sockaddr *) + void *ifa_netmask; // (struct sockaddr *) + // This is a union on Linux. +# ifdef ifa_dstaddr +# undef ifa_dstaddr +# endif + void *ifa_dstaddr; // (struct sockaddr *) + void *ifa_data; +}; + +typedef unsigned __sanitizer_pthread_key_t; + +struct __sanitizer_XDR { + int x_op; + void *x_ops; + uptr x_public; + uptr x_private; + uptr x_base; + unsigned x_handy; +}; + +const int __sanitizer_XDR_ENCODE = 0; +const int __sanitizer_XDR_DECODE = 1; +const int __sanitizer_XDR_FREE = 2; + +struct __sanitizer_passwd { + char *pw_name; + char *pw_passwd; + unsigned int pw_uid; // uid_t + unsigned int pw_gid; // gid_t + char *pw_age; + char *pw_comment; + char *pw_gecos; + char *pw_dir; + char *pw_shell; +}; + +struct __sanitizer_group { + char *gr_name; + char *gr_passwd; + int gr_gid; + char **gr_mem; +}; + +typedef long __sanitizer_time_t; + +struct __sanitizer_timeb { + __sanitizer_time_t time; + unsigned short millitm; + short timezone; + short dstflag; +}; + +struct __sanitizer_ether_addr { + u8 octet[6]; +}; + +struct __sanitizer_tm { + int tm_sec; + int tm_min; + int tm_hour; + int tm_mday; + int tm_mon; + int tm_year; + int tm_wday; + int tm_yday; + int tm_isdst; +}; + +struct __sanitizer_msghdr { + void *msg_name; + unsigned msg_namelen; + struct __sanitizer_iovec *msg_iov; + unsigned msg_iovlen; + void *msg_control; + unsigned msg_controllen; + int msg_flags; +}; +struct __sanitizer_cmsghdr { + unsigned cmsg_len; + int cmsg_level; + int cmsg_type; +}; + +#if SANITIZER_SOLARIS32 && 0 +// FIXME: need to deal with large file and non-large file cases +struct __sanitizer_dirent { + unsigned long long d_ino; + long long d_off; + unsigned short d_reclen; + // more fields that we don't care about +}; +#else +struct __sanitizer_dirent { + unsigned long d_ino; + long d_off; + unsigned short d_reclen; + // more fields that we don't care about +}; +#endif + +struct __sanitizer_dirent64 { + unsigned long long d_ino; + unsigned long long d_off; + unsigned short d_reclen; + // more fields that we don't care about +}; + +typedef long __sanitizer_clock_t; +typedef int __sanitizer_clockid_t; + +// This thing depends on the platform. We are only interested in the upper +// limit. Verified with a compiler assert in .cc. +const int pthread_attr_t_max_sz = 128; +union __sanitizer_pthread_attr_t { + char size[pthread_attr_t_max_sz]; // NOLINT + void *align; +}; + +struct __sanitizer_sigset_t { + // uint32_t * 4 + unsigned int __bits[4]; +}; + +struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; +}; + +using __sanitizer_sighandler_ptr = void (*)(int sig); +using __sanitizer_sigactionhandler_ptr = + void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx); + +struct __sanitizer_sigaction { + int sa_flags; + union { + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; + }; + __sanitizer_sigset_t sa_mask; +#if !defined(_LP64) + int sa_resv[2]; +#endif +}; + +struct __sanitizer_kernel_sigset_t { + u8 sig[8]; +}; + +struct __sanitizer_kernel_sigaction_t { + union { + void (*handler)(int signo); + void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx); + }; + unsigned long sa_flags; + void (*sa_restorer)(void); + __sanitizer_kernel_sigset_t sa_mask; +}; + +extern const uptr sig_ign; +extern const uptr sig_dfl; +extern const uptr sig_err; +extern const uptr sa_siginfo; + +extern int af_inet; +extern int af_inet6; +uptr __sanitizer_in_addr_sz(int af); + +struct __sanitizer_dl_phdr_info { + uptr dlpi_addr; + const char *dlpi_name; + const void *dlpi_phdr; + short dlpi_phnum; +}; + +extern unsigned struct_ElfW_Phdr_sz; + +struct __sanitizer_addrinfo { + int ai_flags; + int ai_family; + int ai_socktype; + int ai_protocol; +#if defined(__sparcv9) + int _ai_pad; +#endif + unsigned ai_addrlen; + char *ai_canonname; + void *ai_addr; + struct __sanitizer_addrinfo *ai_next; +}; + +struct __sanitizer_hostent { + char *h_name; + char **h_aliases; + int h_addrtype; + int h_length; + char **h_addr_list; +}; + +struct __sanitizer_pollfd { + int fd; + short events; + short revents; +}; + +typedef unsigned long __sanitizer_nfds_t; + +struct __sanitizer_glob_t { + uptr gl_pathc; + char **gl_pathv; + uptr gl_offs; + char **gl_pathp; + int gl_pathn; +}; + +extern int glob_nomatch; +extern int glob_altdirfunc; + +extern unsigned path_max; + +struct __sanitizer_wordexp_t { + uptr we_wordc; + char **we_wordv; + uptr we_offs; + char **we_wordp; + int we_wordn; +}; + +typedef void __sanitizer_FILE; +#define SANITIZER_HAS_STRUCT_FILE 0 + +// This simplifies generic code +#define struct_shminfo_sz -1 +#define struct_shm_info_sz -1 +#define shmctl_shm_stat -1 +#define shmctl_ipc_info -1 +#define shmctl_shm_info -1 + +extern int shmctl_ipc_stat; + +extern unsigned struct_utmp_sz; +extern unsigned struct_utmpx_sz; + +extern int map_fixed; + +// ioctl arguments +struct __sanitizer_ifconf { + int ifc_len; + union { + void *ifcu_req; + } ifc_ifcu; +}; + +// <sys/ioccom.h> +#define IOC_NRBITS 8 +#define IOC_TYPEBITS 8 +#define IOC_SIZEBITS 12 +#define IOC_DIRBITS 4 +#undef IOC_NONE +#define IOC_NONE 2U // IOC_VOID +#define IOC_READ 4U // IOC_OUT +#define IOC_WRITE 8U // IOC_IN + +#define IOC_NRMASK ((1 << IOC_NRBITS) - 1) +#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1) +#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1) +#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1) +#define IOC_NRSHIFT 0 +#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS) +#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS) +#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS) + +#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) +#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) +#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) + +#if defined(__sparc__) +// In sparc the 14 bits SIZE field overlaps with the +// least significant bit of DIR, so either IOC_READ or +// IOC_WRITE shall be 1 in order to get a non-zero SIZE. +#define IOC_SIZE(nr) \ + ((((((nr) >> 29) & 0x7) & (4U | 2U)) == 0) ? 0 : (((nr) >> 16) & 0x3fff)) +#else +#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) +#endif + +extern unsigned struct_ifreq_sz; +extern unsigned struct_termios_sz; +extern unsigned struct_winsize_sz; + +extern unsigned struct_sioc_sg_req_sz; +extern unsigned struct_sioc_vif_req_sz; + +// ioctl request identifiers + +// A special value to mark ioctls that are not present on the target platform, +// when it can not be determined without including any system headers. +extern const unsigned IOCTL_NOT_PRESENT; + +extern unsigned IOCTL_FIOASYNC; +extern unsigned IOCTL_FIOCLEX; +extern unsigned IOCTL_FIOGETOWN; +extern unsigned IOCTL_FIONBIO; +extern unsigned IOCTL_FIONCLEX; +extern unsigned IOCTL_FIOSETOWN; +extern unsigned IOCTL_SIOCADDMULTI; +extern unsigned IOCTL_SIOCATMARK; +extern unsigned IOCTL_SIOCDELMULTI; +extern unsigned IOCTL_SIOCGIFADDR; +extern unsigned IOCTL_SIOCGIFBRDADDR; +extern unsigned IOCTL_SIOCGIFCONF; +extern unsigned IOCTL_SIOCGIFDSTADDR; +extern unsigned IOCTL_SIOCGIFFLAGS; +extern unsigned IOCTL_SIOCGIFMETRIC; +extern unsigned IOCTL_SIOCGIFMTU; +extern unsigned IOCTL_SIOCGIFNETMASK; +extern unsigned IOCTL_SIOCGPGRP; +extern unsigned IOCTL_SIOCSIFADDR; +extern unsigned IOCTL_SIOCSIFBRDADDR; +extern unsigned IOCTL_SIOCSIFDSTADDR; +extern unsigned IOCTL_SIOCSIFFLAGS; +extern unsigned IOCTL_SIOCSIFMETRIC; +extern unsigned IOCTL_SIOCSIFMTU; +extern unsigned IOCTL_SIOCSIFNETMASK; +extern unsigned IOCTL_SIOCSPGRP; +extern unsigned IOCTL_TIOCEXCL; +extern unsigned IOCTL_TIOCGETD; +extern unsigned IOCTL_TIOCGPGRP; +extern unsigned IOCTL_TIOCGWINSZ; +extern unsigned IOCTL_TIOCMBIC; +extern unsigned IOCTL_TIOCMBIS; +extern unsigned IOCTL_TIOCMGET; +extern unsigned IOCTL_TIOCMSET; +extern unsigned IOCTL_TIOCNOTTY; +extern unsigned IOCTL_TIOCNXCL; +extern unsigned IOCTL_TIOCOUTQ; +extern unsigned IOCTL_TIOCPKT; +extern unsigned IOCTL_TIOCSCTTY; +extern unsigned IOCTL_TIOCSETD; +extern unsigned IOCTL_TIOCSPGRP; +extern unsigned IOCTL_TIOCSTI; +extern unsigned IOCTL_TIOCSWINSZ; +extern unsigned IOCTL_MTIOCGET; +extern unsigned IOCTL_MTIOCTOP; + +extern const int si_SEGV_MAPERR; +extern const int si_SEGV_ACCERR; +} // namespace __sanitizer + +#define CHECK_TYPE_SIZE(TYPE) \ + COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE)) + +#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \ + offsetof(CLASS, MEMBER)) + +// For sigaction, which is a function and struct at the same time, +// and thus requires explicit "struct" in sizeof() expression. +#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((struct CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ + offsetof(struct CLASS, MEMBER)) + +#endif // SANITIZER_SOLARIS + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc index 8d3128ae199d..1fad71f3582b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc @@ -17,6 +17,7 @@ #if SANITIZER_POSIX #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_libc.h" #include "sanitizer_posix.h" #include "sanitizer_procmaps.h" @@ -294,18 +295,22 @@ bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) { return false; } -SignalContext SignalContext::Create(void *siginfo, void *context) { - auto si = (siginfo_t *)siginfo; - uptr addr = (uptr)si->si_addr; - uptr pc, sp, bp; - GetPcSpBp(context, &pc, &sp, &bp); - WriteFlag write_flag = GetWriteFlag(context); - bool is_memory_access = si->si_signo == SIGSEGV; - return SignalContext(context, addr, pc, sp, bp, is_memory_access, write_flag); +uptr SignalContext::GetAddress() const { + auto si = static_cast<const siginfo_t *>(siginfo); + return (uptr)si->si_addr; } -const char *DescribeSignalOrException(int signo) { - switch (signo) { +bool SignalContext::IsMemoryAccess() const { + auto si = static_cast<const siginfo_t *>(siginfo); + return si->si_signo == SIGSEGV; +} + +int SignalContext::GetType() const { + return static_cast<const siginfo_t *>(siginfo)->si_signo; +} + +const char *SignalContext::Describe() const { + switch (GetType()) { case SIGFPE: return "FPE"; case SIGILL: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h index e7d37cbf0882..adef08248004 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h @@ -16,7 +16,9 @@ // ----------- ATTENTION ------------- // This header should NOT include any other headers from sanitizer runtime. #include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_posix.h" +#include "sanitizer_platform_limits_solaris.h" #if !SANITIZER_POSIX // Make it hard to accidentally use any of functions declared in this file: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc index e113fb1093d4..db41cad6c318 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -18,7 +18,9 @@ #include "sanitizer_common.h" #include "sanitizer_flags.h" +#include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_posix.h" +#include "sanitizer_platform_limits_solaris.h" #include "sanitizer_posix.h" #include "sanitizer_procmaps.h" #include "sanitizer_stacktrace.h" @@ -61,7 +63,11 @@ void ReleaseMemoryPagesToOS(uptr beg, uptr end) { uptr beg_aligned = RoundUpTo(beg, page_size); uptr end_aligned = RoundDownTo(end, page_size); if (beg_aligned < end_aligned) - madvise((void*)beg_aligned, end_aligned - beg_aligned, MADV_DONTNEED); + // In the default Solaris compilation environment, madvise() is declared + // to take a caddr_t arg; casting it to void * results in an invalid + // conversion error, so use char * instead. + madvise((char *)beg_aligned, end_aligned - beg_aligned, + SANITIZER_MADVISE_DONTNEED); } void NoHugePagesInRegion(uptr addr, uptr size) { @@ -213,6 +219,53 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) { MaybeInstallSigaction(SIGFPE, handler); MaybeInstallSigaction(SIGILL, handler); } + +bool SignalContext::IsStackOverflow() const { + // Access at a reasonable offset above SP, or slightly below it (to account + // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is + // probably a stack overflow. +#ifdef __s390__ + // On s390, the fault address in siginfo points to start of the page, not + // to the precise word that was accessed. Mask off the low bits of sp to + // take it into account. + bool IsStackAccess = addr >= (sp & ~0xFFF) && addr < sp + 0xFFFF; +#else + bool IsStackAccess = addr + 512 > sp && addr < sp + 0xFFFF; +#endif + +#if __powerpc__ + // Large stack frames can be allocated with e.g. + // lis r0,-10000 + // stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000 + // If the store faults then sp will not have been updated, so test above + // will not work, because the fault address will be more than just "slightly" + // below sp. + if (!IsStackAccess && IsAccessibleMemoryRange(pc, 4)) { + u32 inst = *(unsigned *)pc; + u32 ra = (inst >> 16) & 0x1F; + u32 opcd = inst >> 26; + u32 xo = (inst >> 1) & 0x3FF; + // Check for store-with-update to sp. The instructions we accept are: + // stbu rs,d(ra) stbux rs,ra,rb + // sthu rs,d(ra) sthux rs,ra,rb + // stwu rs,d(ra) stwux rs,ra,rb + // stdu rs,ds(ra) stdux rs,ra,rb + // where ra is r1 (the stack pointer). + if (ra == 1 && + (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 || + (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181)))) + IsStackAccess = true; + } +#endif // __powerpc__ + + // We also check si_code to filter out SEGV caused by something else other + // then hitting the guard page or unmapped memory, like, for example, + // unaligned memory access. + auto si = static_cast<const siginfo_t *>(siginfo); + return IsStackAccess && + (si->si_code == si_SEGV_MAPERR || si->si_code == si_SEGV_ACCERR); +} + #endif // SANITIZER_GO bool IsAccessibleMemoryRange(uptr beg, uptr size) { @@ -289,6 +342,46 @@ void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { return (void *)p; } +uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { + // We don't pass `name` along because, when you enable `decorate_proc_maps` + // AND actually use a named mapping AND are using a sanitizer intercepting + // `open` (e.g. TSAN, ESAN), then you'll get a failure during initialization. + // TODO(flowerhack): Fix the implementation of GetNamedMappingFd to solve + // this problem. + if (fixed_addr) { + base_ = MmapFixedNoAccess(fixed_addr, size); + } else { + base_ = MmapNoAccess(size); + } + size_ = size; + name_ = name; + (void)os_handle_; // unsupported + return reinterpret_cast<uptr>(base_); +} + +// Uses fixed_addr for now. +// Will use offset instead once we've implemented this function for real. +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { + return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size)); +} + +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { + return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size)); +} + +void ReservedAddressRange::Unmap(uptr addr, uptr size) { + void* addr_as_void = reinterpret_cast<void*>(addr); + uptr base_as_uptr = reinterpret_cast<uptr>(base_); + // Only unmap at the beginning or end of the range. + CHECK((addr_as_void == base_) || (addr + size == base_as_uptr + size_)); + CHECK_LE(size, size_); + UnmapOrDie(reinterpret_cast<void*>(addr), size); + if (addr_as_void == base_) { + base_ = reinterpret_cast<void*>(addr + size); + } + size_ = size_ - size; +} + void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { int fd = name ? GetNamedMappingFd(name, size) : -1; unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc index 99b7ff1b55cf..5c2360043b07 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc @@ -28,8 +28,6 @@ namespace __sanitizer { -StaticSpinMutex CommonSanitizerReportMutex; - static int AppendChar(char **buff, const char *buff_end, char c) { if (*buff < buff_end) { **buff = c; @@ -229,19 +227,16 @@ static void CallPrintfAndReportCallback(const char *str) { PrintfAndReportCallback(str); } -static void SharedPrintfCode(bool append_pid, const char *format, - va_list args) { +static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid, + char *local_buffer, + int buffer_size, + const char *format, + va_list args) { va_list args2; va_copy(args2, args); const int kLen = 16 * 1024; - // |local_buffer| is small enough not to overflow the stack and/or violate - // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other - // hand, the bigger the buffer is, the more the chance the error report will - // fit into it. - char local_buffer[400]; int needed_length; char *buffer = local_buffer; - int buffer_size = ARRAY_SIZE(local_buffer); // First try to print a message using a local buffer, and then fall back to // mmaped buffer. for (int use_mmap = 0; use_mmap < 2; use_mmap++) { @@ -259,7 +254,9 @@ static void SharedPrintfCode(bool append_pid, const char *format, RAW_CHECK_MSG(needed_length < kLen, \ "Buffer in Report is too short!\n"); \ } - if (append_pid) { + // Fuchsia's logging infrastructure always keeps track of the logging + // process, thread, and timestamp, so never prepend such information. + if (!SANITIZER_FUCHSIA && append_pid) { int pid = internal_getpid(); const char *exe_name = GetProcessName(); if (common_flags()->log_exe_name && exe_name) { @@ -267,9 +264,8 @@ static void SharedPrintfCode(bool append_pid, const char *format, "==%s", exe_name); CHECK_NEEDED_LENGTH } - needed_length += internal_snprintf(buffer + needed_length, - buffer_size - needed_length, - "==%d==", pid); + needed_length += internal_snprintf( + buffer + needed_length, buffer_size - needed_length, "==%d==", pid); CHECK_NEEDED_LENGTH } needed_length += VSNPrintf(buffer + needed_length, @@ -292,6 +288,17 @@ static void SharedPrintfCode(bool append_pid, const char *format, va_end(args2); } +static void NOINLINE SharedPrintfCode(bool append_pid, const char *format, + va_list args) { + // |local_buffer| is small enough not to overflow the stack and/or violate + // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other + // hand, the bigger the buffer is, the more the chance the error report will + // fit into it. + char local_buffer[400]; + SharedPrintfCodeNoBuffer(append_pid, local_buffer, ARRAY_SIZE(local_buffer), + format, args); +} + FORMAT(1, 2) void Printf(const char *format, ...) { va_list args; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h index 06d072b4dc77..ea2cb7a5c68b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h @@ -14,22 +14,19 @@ #ifndef SANITIZER_PROCMAPS_H #define SANITIZER_PROCMAPS_H +#include "sanitizer_platform.h" + +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_MAC || SANITIZER_SOLARIS + #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" +#include "sanitizer_linux.h" +#include "sanitizer_mac.h" #include "sanitizer_mutex.h" namespace __sanitizer { -#if SANITIZER_FREEBSD || SANITIZER_LINUX -struct ProcSelfMapsBuff { - char *data; - uptr mmaped_size; - uptr len; -}; - -// Reads process memory map in an OS-specific way. -void ReadProcMaps(ProcSelfMapsBuff *proc_maps); -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX // Memory protection masks. static const uptr kProtectionRead = 1; @@ -37,15 +34,20 @@ static const uptr kProtectionWrite = 2; static const uptr kProtectionExecute = 4; static const uptr kProtectionShared = 8; -struct MemoryMappedSegment { +struct MemoryMappedSegmentData; + +class MemoryMappedSegment { + public: MemoryMappedSegment(char *buff = nullptr, uptr size = 0) - : filename(buff), filename_size(size) {} + : filename(buff), filename_size(size), data_(nullptr) {} ~MemoryMappedSegment() {} - bool IsReadable() { return protection & kProtectionRead; } - bool IsWritable() { return protection & kProtectionWrite; } - bool IsExecutable() { return protection & kProtectionExecute; } - bool IsShared() { return protection & kProtectionShared; } + bool IsReadable() const { return protection & kProtectionRead; } + bool IsWritable() const { return protection & kProtectionWrite; } + bool IsExecutable() const { return protection & kProtectionExecute; } + bool IsShared() const { return protection & kProtectionShared; } + + void AddAddressRanges(LoadedModule *module); uptr start; uptr end; @@ -55,6 +57,12 @@ struct MemoryMappedSegment { uptr protection; ModuleArch arch; u8 uuid[kModuleUUIDSize]; + + private: + friend class MemoryMappingLayout; + + // This field is assigned and owned by MemoryMappingLayout if needed + MemoryMappedSegmentData *data_; }; class MemoryMappingLayout { @@ -69,42 +77,14 @@ class MemoryMappingLayout { static void CacheMemoryMappings(); // Adds all mapped objects into a vector. - void DumpListOfModules(InternalMmapVector<LoadedModule> *modules); + void DumpListOfModules(InternalMmapVectorNoCtor<LoadedModule> *modules); private: void LoadFromCache(); - // FIXME: Hide implementation details for different platforms in - // platform-specific files. -# if SANITIZER_FREEBSD || SANITIZER_LINUX - ProcSelfMapsBuff proc_self_maps_; - const char *current_; - - // Static mappings cache. - static ProcSelfMapsBuff cached_proc_self_maps_; - static StaticSpinMutex cache_lock_; // protects cached_proc_self_maps_. -# elif SANITIZER_MAC - template <u32 kLCSegment, typename SegmentCommand> - bool NextSegmentLoad(MemoryMappedSegment *segment); - int current_image_; - u32 current_magic_; - u32 current_filetype_; - ModuleArch current_arch_; - u8 current_uuid_[kModuleUUIDSize]; - int current_load_cmd_count_; - char *current_load_cmd_addr_; - bool current_instrumented_; -# endif + MemoryMappingLayoutData data_; }; -typedef void (*fill_profile_f)(uptr start, uptr rss, bool file, - /*out*/uptr *stats, uptr stats_size); - -// Parse the contents of /proc/self/smaps and generate a memory profile. -// |cb| is a tool-specific callback that fills the |stats| array containing -// |stats_size| elements. -void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size); - // Returns code range for the specified module. bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end); @@ -115,4 +95,6 @@ uptr ParseHex(const char **p); } // namespace __sanitizer +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || + // SANITIZER_MAC || SANITIZER_SOLARIS #endif // SANITIZER_PROCMAPS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc index b95f301a437d..0cd3e2458d87 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -12,7 +12,8 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_placement_new.h" @@ -20,9 +21,8 @@ namespace __sanitizer { -// Linker initialized. -ProcSelfMapsBuff MemoryMappingLayout::cached_proc_self_maps_; -StaticSpinMutex MemoryMappingLayout::cache_lock_; // Linker initialized. +static ProcSelfMapsBuff cached_proc_self_maps; +static StaticSpinMutex cache_lock; static int TranslateDigit(char c) { if (c >= '0' && c <= '9') @@ -64,60 +64,60 @@ uptr ParseHex(const char **p) { return ParseNumber(p, 16); } +void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) { + // data_ should be unused on this platform + CHECK(!data_); + module->addAddressRange(start, end, IsExecutable(), IsWritable()); +} + MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { - ReadProcMaps(&proc_self_maps_); - if (cache_enabled) { - if (proc_self_maps_.mmaped_size == 0) { - LoadFromCache(); - CHECK_GT(proc_self_maps_.len, 0); - } - } else { - CHECK_GT(proc_self_maps_.mmaped_size, 0); - } - Reset(); // FIXME: in the future we may want to cache the mappings on demand only. if (cache_enabled) CacheMemoryMappings(); + + // Read maps after the cache update to capture the maps/unmaps happening in + // the process of updating. + ReadProcMaps(&data_.proc_self_maps); + if (cache_enabled && data_.proc_self_maps.mmaped_size == 0) + LoadFromCache(); + CHECK_GT(data_.proc_self_maps.mmaped_size, 0); + CHECK_GT(data_.proc_self_maps.len, 0); + + Reset(); } MemoryMappingLayout::~MemoryMappingLayout() { // Only unmap the buffer if it is different from the cached one. Otherwise // it will be unmapped when the cache is refreshed. - if (proc_self_maps_.data != cached_proc_self_maps_.data) { - UnmapOrDie(proc_self_maps_.data, proc_self_maps_.mmaped_size); - } + if (data_.proc_self_maps.data != cached_proc_self_maps.data) + UnmapOrDie(data_.proc_self_maps.data, data_.proc_self_maps.mmaped_size); } void MemoryMappingLayout::Reset() { - current_ = proc_self_maps_.data; + data_.current = data_.proc_self_maps.data; } // static void MemoryMappingLayout::CacheMemoryMappings() { - SpinMutexLock l(&cache_lock_); + ProcSelfMapsBuff new_proc_self_maps; + ReadProcMaps(&new_proc_self_maps); // Don't invalidate the cache if the mappings are unavailable. - ProcSelfMapsBuff old_proc_self_maps; - old_proc_self_maps = cached_proc_self_maps_; - ReadProcMaps(&cached_proc_self_maps_); - if (cached_proc_self_maps_.mmaped_size == 0) { - cached_proc_self_maps_ = old_proc_self_maps; - } else { - if (old_proc_self_maps.mmaped_size) { - UnmapOrDie(old_proc_self_maps.data, - old_proc_self_maps.mmaped_size); - } - } + if (new_proc_self_maps.mmaped_size == 0) + return; + SpinMutexLock l(&cache_lock); + if (cached_proc_self_maps.mmaped_size) + UnmapOrDie(cached_proc_self_maps.data, cached_proc_self_maps.mmaped_size); + cached_proc_self_maps = new_proc_self_maps; } void MemoryMappingLayout::LoadFromCache() { - SpinMutexLock l(&cache_lock_); - if (cached_proc_self_maps_.data) { - proc_self_maps_ = cached_proc_self_maps_; - } + SpinMutexLock l(&cache_lock); + if (cached_proc_self_maps.data) + data_.proc_self_maps = cached_proc_self_maps; } void MemoryMappingLayout::DumpListOfModules( - InternalMmapVector<LoadedModule> *modules) { + InternalMmapVectorNoCtor<LoadedModule> *modules) { Reset(); InternalScopedString module_name(kMaxPathLength); MemoryMappedSegment segment(module_name.data(), module_name.size()); @@ -139,8 +139,7 @@ void MemoryMappingLayout::DumpListOfModules( uptr base_address = (i ? segment.start : 0) - segment.offset; LoadedModule cur_module; cur_module.set(cur_name, base_address); - cur_module.addAddressRange(segment.start, segment.end, - segment.IsExecutable(), segment.IsWritable()); + segment.AddAddressRanges(&cur_module); modules->push_back(cur_module); } } @@ -171,4 +170,5 @@ void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc index f0cdbeb4483a..c26a6d47ea75 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc @@ -7,18 +7,22 @@ // //===----------------------------------------------------------------------===// // -// Information about the process mappings (FreeBSD-specific parts). +// Information about the process mappings (FreeBSD and NetBSD-specific parts). //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD #include "sanitizer_common.h" +#if SANITIZER_FREEBSD #include "sanitizer_freebsd.h" +#endif #include "sanitizer_procmaps.h" #include <unistd.h> #include <sys/sysctl.h> +#if SANITIZER_FREEBSD #include <sys/user.h> +#endif // Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode. #if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32) @@ -31,16 +35,30 @@ namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - const int Mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid() }; + const int Mib[] = { +#if SANITIZER_FREEBSD + CTL_KERN, + KERN_PROC, + KERN_PROC_VMMAP, + getpid() +#else + CTL_VM, + VM_PROC, + VM_PROC_MAP, + getpid(), + sizeof(struct kinfo_vmentry) +#endif + }; + size_t Size = 0; - int Err = sysctl(Mib, 4, NULL, &Size, NULL, 0); + int Err = sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0); CHECK_EQ(Err, 0); CHECK_GT(Size, 0); size_t MmapedSize = Size * 4 / 3; void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()"); Size = MmapedSize; - Err = sysctl(Mib, 4, VmMap, &Size, NULL, 0); + Err = sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0); CHECK_EQ(Err, 0); proc_maps->data = (char*)VmMap; @@ -49,9 +67,9 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { - char *last = proc_self_maps_.data + proc_self_maps_.len; - if (current_ >= last) return false; - struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry*)current_; + char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; + if (data_.current >= last) return false; + struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry *)data_.current; segment->start = (uptr)VmEntry->kve_start; segment->end = (uptr)VmEntry->kve_end; @@ -71,11 +89,15 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { VmEntry->kve_path); } - current_ += VmEntry->kve_structsize; +#if SANITIZER_FREEBSD + data_.current += VmEntry->kve_structsize; +#else + data_.current += sizeof(*VmEntry); +#endif return true; } } // namespace __sanitizer -#endif // SANITIZER_FREEBSD +#endif // SANITIZER_FREEBSD || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc index 1bcad2bf70e6..633e9393cce0 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -18,8 +18,12 @@ namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - ReadFileToBuffer("/proc/self/maps", &proc_maps->data, &proc_maps->mmaped_size, - &proc_maps->len); + if (!ReadFileToBuffer("/proc/self/maps", &proc_maps->data, + &proc_maps->mmaped_size, &proc_maps->len)) { + proc_maps->data = nullptr; + proc_maps->mmaped_size = 0; + proc_maps->len = 0; + } } static bool IsOneOf(char c, char c1, char c2) { @@ -27,48 +31,48 @@ static bool IsOneOf(char c, char c1, char c2) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { - char *last = proc_self_maps_.data + proc_self_maps_.len; - if (current_ >= last) return false; - char *next_line = (char*)internal_memchr(current_, '\n', last - current_); + char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; + if (data_.current >= last) return false; + char *next_line = + (char *)internal_memchr(data_.current, '\n', last - data_.current); if (next_line == 0) next_line = last; // Example: 08048000-08056000 r-xp 00000000 03:0c 64593 /foo/bar - segment->start = ParseHex(¤t_); - CHECK_EQ(*current_++, '-'); - segment->end = ParseHex(¤t_); - CHECK_EQ(*current_++, ' '); - CHECK(IsOneOf(*current_, '-', 'r')); + segment->start = ParseHex(&data_.current); + CHECK_EQ(*data_.current++, '-'); + segment->end = ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ' '); + CHECK(IsOneOf(*data_.current, '-', 'r')); segment->protection = 0; - if (*current_++ == 'r') segment->protection |= kProtectionRead; - CHECK(IsOneOf(*current_, '-', 'w')); - if (*current_++ == 'w') segment->protection |= kProtectionWrite; - CHECK(IsOneOf(*current_, '-', 'x')); - if (*current_++ == 'x') segment->protection |= kProtectionExecute; - CHECK(IsOneOf(*current_, 's', 'p')); - if (*current_++ == 's') segment->protection |= kProtectionShared; - CHECK_EQ(*current_++, ' '); - segment->offset = ParseHex(¤t_); - CHECK_EQ(*current_++, ' '); - ParseHex(¤t_); - CHECK_EQ(*current_++, ':'); - ParseHex(¤t_); - CHECK_EQ(*current_++, ' '); - while (IsDecimal(*current_)) - current_++; + if (*data_.current++ == 'r') segment->protection |= kProtectionRead; + CHECK(IsOneOf(*data_.current, '-', 'w')); + if (*data_.current++ == 'w') segment->protection |= kProtectionWrite; + CHECK(IsOneOf(*data_.current, '-', 'x')); + if (*data_.current++ == 'x') segment->protection |= kProtectionExecute; + CHECK(IsOneOf(*data_.current, 's', 'p')); + if (*data_.current++ == 's') segment->protection |= kProtectionShared; + CHECK_EQ(*data_.current++, ' '); + segment->offset = ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ' '); + ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ':'); + ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ' '); + while (IsDecimal(*data_.current)) data_.current++; // Qemu may lack the trailing space. // https://github.com/google/sanitizers/issues/160 - // CHECK_EQ(*current_++, ' '); + // CHECK_EQ(*data_.current++, ' '); // Skip spaces. - while (current_ < next_line && *current_ == ' ') - current_++; + while (data_.current < next_line && *data_.current == ' ') data_.current++; // Fill in the filename. if (segment->filename) { - uptr len = Min((uptr)(next_line - current_), segment->filename_size - 1); - internal_strncpy(segment->filename, current_, len); + uptr len = + Min((uptr)(next_line - data_.current), segment->filename_size - 1); + internal_strncpy(segment->filename, data_.current, len); segment->filename[len] = 0; } - current_ = next_line + 1; + data_.current = next_line + 1; return true; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 560451a16d90..0167ab18ba13 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -36,6 +36,51 @@ namespace __sanitizer { +// Contains information used to iterate through sections. +struct MemoryMappedSegmentData { + char name[kMaxSegName]; + uptr nsects; + const char *current_load_cmd_addr; + u32 lc_type; + uptr base_virt_addr; + uptr addr_mask; +}; + +template <typename Section> +static void NextSectionLoad(LoadedModule *module, MemoryMappedSegmentData *data, + bool isWritable) { + const Section *sc = (const Section *)data->current_load_cmd_addr; + data->current_load_cmd_addr += sizeof(Section); + + uptr sec_start = (sc->addr & data->addr_mask) + data->base_virt_addr; + uptr sec_end = sec_start + sc->size; + module->addAddressRange(sec_start, sec_end, /*executable=*/false, isWritable, + sc->sectname); +} + +void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) { + // Don't iterate over sections when the caller hasn't set up the + // data pointer, when there are no sections, or when the segment + // is executable. Avoid iterating over executable sections because + // it will confuse libignore, and because the extra granularity + // of information is not needed by any sanitizers. + if (!data_ || !data_->nsects || IsExecutable()) { + module->addAddressRange(start, end, IsExecutable(), IsWritable(), + data_ ? data_->name : nullptr); + return; + } + + do { + if (data_->lc_type == LC_SEGMENT) { + NextSectionLoad<struct section>(module, data_, IsWritable()); +#ifdef MH_MAGIC_64 + } else if (data_->lc_type == LC_SEGMENT_64) { + NextSectionLoad<struct section_64>(module, data_, IsWritable()); +#endif + } + } while (--data_->nsects); +} + MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { Reset(); } @@ -63,13 +108,13 @@ void MemoryMappingLayout::Reset() { // _dyld_image_count is thread-unsafe. We need to register callbacks for // adding and removing images which will invalidate the MemoryMappingLayout // state. - current_image_ = _dyld_image_count(); - current_load_cmd_count_ = -1; - current_load_cmd_addr_ = 0; - current_magic_ = 0; - current_filetype_ = 0; - current_arch_ = kModuleArchUnknown; - internal_memset(current_uuid_, 0, kModuleUUIDSize); + data_.current_image = _dyld_image_count(); + data_.current_load_cmd_count = -1; + data_.current_load_cmd_addr = 0; + data_.current_magic = 0; + data_.current_filetype = 0; + data_.current_arch = kModuleArchUnknown; + internal_memset(data_.current_uuid, 0, kModuleUUIDSize); } // The dyld load address should be unchanged throughout process execution, @@ -138,39 +183,57 @@ const mach_header *get_dyld_hdr() { // segment. // Note that the segment addresses are not necessarily sorted. template <u32 kLCSegment, typename SegmentCommand> -bool MemoryMappingLayout::NextSegmentLoad(MemoryMappedSegment *segment) { - const char *lc = current_load_cmd_addr_; - current_load_cmd_addr_ += ((const load_command *)lc)->cmdsize; +static bool NextSegmentLoad(MemoryMappedSegment *segment, +MemoryMappedSegmentData *seg_data, MemoryMappingLayoutData &layout_data) { + const char *lc = layout_data.current_load_cmd_addr; + layout_data.current_load_cmd_addr += ((const load_command *)lc)->cmdsize; if (((const load_command *)lc)->cmd == kLCSegment) { const SegmentCommand* sc = (const SegmentCommand *)lc; - - if (current_image_ == kDyldImageIdx) { + uptr base_virt_addr, addr_mask; + if (layout_data.current_image == kDyldImageIdx) { + base_virt_addr = (uptr)get_dyld_hdr(); // vmaddr is masked with 0xfffff because on macOS versions < 10.12, // it contains an absolute address rather than an offset for dyld. // To make matters even more complicated, this absolute address // isn't actually the absolute segment address, but the offset portion // of the address is accurate when combined with the dyld base address, // and the mask will give just this offset. - segment->start = (sc->vmaddr & 0xfffff) + (uptr)get_dyld_hdr(); - segment->end = (sc->vmaddr & 0xfffff) + sc->vmsize + (uptr)get_dyld_hdr(); + addr_mask = 0xfffff; } else { - const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_); - segment->start = sc->vmaddr + dlloff; - segment->end = sc->vmaddr + sc->vmsize + dlloff; + base_virt_addr = + (uptr)_dyld_get_image_vmaddr_slide(layout_data.current_image); + addr_mask = ~0; + } + + segment->start = (sc->vmaddr & addr_mask) + base_virt_addr; + segment->end = segment->start + sc->vmsize; + // Most callers don't need section information, so only fill this struct + // when required. + if (seg_data) { + seg_data->nsects = sc->nsects; + seg_data->current_load_cmd_addr = + (const char *)lc + sizeof(SegmentCommand); + seg_data->lc_type = kLCSegment; + seg_data->base_virt_addr = base_virt_addr; + seg_data->addr_mask = addr_mask; + internal_strncpy(seg_data->name, sc->segname, + ARRAY_SIZE(seg_data->name)); } // Return the initial protection. segment->protection = sc->initprot; - segment->offset = - (current_filetype_ == /*MH_EXECUTE*/ 0x2) ? sc->vmaddr : sc->fileoff; + segment->offset = (layout_data.current_filetype == + /*MH_EXECUTE*/ 0x2) + ? sc->vmaddr + : sc->fileoff; if (segment->filename) { - const char *src = (current_image_ == kDyldImageIdx) + const char *src = (layout_data.current_image == kDyldImageIdx) ? kDyldPath - : _dyld_get_image_name(current_image_); + : _dyld_get_image_name(layout_data.current_image); internal_strncpy(segment->filename, src, segment->filename_size); } - segment->arch = current_arch_; - internal_memcpy(segment->uuid, current_uuid_, kModuleUUIDSize); + segment->arch = layout_data.current_arch; + internal_memcpy(segment->uuid, layout_data.current_uuid, kModuleUUIDSize); return true; } return false; @@ -202,7 +265,7 @@ ModuleArch ModuleArchFromCpuType(cpu_type_t cputype, cpu_subtype_t cpusubtype) { } static const load_command *NextCommand(const load_command *lc) { - return (const load_command *)((char *)lc + lc->cmdsize); + return (const load_command *)((const char *)lc + lc->cmdsize); } static void FindUUID(const load_command *first_lc, u8 *uuid_output) { @@ -232,50 +295,53 @@ static bool IsModuleInstrumented(const load_command *first_lc) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { - for (; current_image_ >= kDyldImageIdx; current_image_--) { - const mach_header *hdr = (current_image_ == kDyldImageIdx) + for (; data_.current_image >= kDyldImageIdx; data_.current_image--) { + const mach_header *hdr = (data_.current_image == kDyldImageIdx) ? get_dyld_hdr() - : _dyld_get_image_header(current_image_); + : _dyld_get_image_header(data_.current_image); if (!hdr) continue; - if (current_load_cmd_count_ < 0) { + if (data_.current_load_cmd_count < 0) { // Set up for this image; - current_load_cmd_count_ = hdr->ncmds; - current_magic_ = hdr->magic; - current_filetype_ = hdr->filetype; - current_arch_ = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype); - switch (current_magic_) { + data_.current_load_cmd_count = hdr->ncmds; + data_.current_magic = hdr->magic; + data_.current_filetype = hdr->filetype; + data_.current_arch = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype); + switch (data_.current_magic) { #ifdef MH_MAGIC_64 case MH_MAGIC_64: { - current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header_64); + data_.current_load_cmd_addr = + (const char *)hdr + sizeof(mach_header_64); break; } #endif case MH_MAGIC: { - current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header); + data_.current_load_cmd_addr = (const char *)hdr + sizeof(mach_header); break; } default: { continue; } } - FindUUID((const load_command *)current_load_cmd_addr_, ¤t_uuid_[0]); - current_instrumented_ = - IsModuleInstrumented((const load_command *)current_load_cmd_addr_); + FindUUID((const load_command *)data_.current_load_cmd_addr, + data_.current_uuid); + data_.current_instrumented = IsModuleInstrumented( + (const load_command *)data_.current_load_cmd_addr); } - for (; current_load_cmd_count_ >= 0; current_load_cmd_count_--) { - switch (current_magic_) { - // current_magic_ may be only one of MH_MAGIC, MH_MAGIC_64. + for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) { + switch (data_.current_magic) { + // data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64. #ifdef MH_MAGIC_64 case MH_MAGIC_64: { if (NextSegmentLoad<LC_SEGMENT_64, struct segment_command_64>( - segment)) + segment, segment->data_, data_)) return true; break; } #endif case MH_MAGIC: { - if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(segment)) + if (NextSegmentLoad<LC_SEGMENT, struct segment_command>( + segment, segment->data_, data_)) return true; break; } @@ -288,11 +354,13 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { } void MemoryMappingLayout::DumpListOfModules( - InternalMmapVector<LoadedModule> *modules) { + InternalMmapVectorNoCtor<LoadedModule> *modules) { Reset(); InternalScopedString module_name(kMaxPathLength); MemoryMappedSegment segment(module_name.data(), kMaxPathLength); - for (uptr i = 0; Next(&segment); i++) { + MemoryMappedSegmentData data; + segment.data_ = &data; + while (Next(&segment)) { if (segment.filename[0] == '\0') continue; LoadedModule *cur_module = nullptr; if (!modules->empty() && @@ -302,10 +370,9 @@ void MemoryMappingLayout::DumpListOfModules( modules->push_back(LoadedModule()); cur_module = &modules->back(); cur_module->set(segment.filename, segment.start, segment.arch, - segment.uuid, current_instrumented_); + segment.uuid, data_.current_instrumented); } - cur_module->addAddressRange(segment.start, segment.end, - segment.IsExecutable(), segment.IsWritable()); + segment.AddAddressRanges(cur_module); } } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc new file mode 100644 index 000000000000..bfe83170f4e2 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc @@ -0,0 +1,57 @@ +//===-- sanitizer_procmaps_solaris.cc -------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Information about the process mappings (Solaris-specific parts). +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_SOLARIS +#include "sanitizer_common.h" +#include "sanitizer_procmaps.h" + +#include <procfs.h> +#include <limits.h> + +namespace __sanitizer { + +void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { + ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, &proc_maps->mmaped_size, + &proc_maps->len); +} + +bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; + if (data_.current >= last) return false; + + prxmap_t *xmapentry = (prxmap_t*)data_.current; + + segment->start = (uptr)xmapentry->pr_vaddr; + segment->end = (uptr)(xmapentry->pr_vaddr + xmapentry->pr_size); + segment->offset = (uptr)xmapentry->pr_offset; + + segment->protection = 0; + if ((xmapentry->pr_mflags & MA_READ) != 0) + segment->protection |= kProtectionRead; + if ((xmapentry->pr_mflags & MA_WRITE) != 0) + segment->protection |= kProtectionWrite; + if ((xmapentry->pr_mflags & MA_EXEC) != 0) + segment->protection |= kProtectionExecute; + + if (segment->filename != NULL && segment->filename_size > 0) { + internal_snprintf(segment->filename, + Min(segment->filename_size, (uptr)PATH_MAX), "%s", + xmapentry->pr_mapname); + } + + data_.current += sizeof(prxmap_t); + + return true; +} + +} // namespace __sanitizer + +#endif // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h index db38867ced28..886445a27322 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h @@ -87,15 +87,14 @@ class Quarantine { // is zero (it allows us to perform just one atomic read per Put() call). CHECK((size == 0 && cache_size == 0) || cache_size != 0); - atomic_store(&max_size_, size, memory_order_relaxed); - atomic_store(&min_size_, size / 10 * 9, - memory_order_relaxed); // 90% of max size. - atomic_store(&max_cache_size_, cache_size, memory_order_relaxed); + atomic_store_relaxed(&max_size_, size); + atomic_store_relaxed(&min_size_, size / 10 * 9); // 90% of max size. + atomic_store_relaxed(&max_cache_size_, cache_size); } - uptr GetSize() const { return atomic_load(&max_size_, memory_order_relaxed); } + uptr GetSize() const { return atomic_load_relaxed(&max_size_); } uptr GetCacheSize() const { - return atomic_load(&max_cache_size_, memory_order_relaxed); + return atomic_load_relaxed(&max_cache_size_); } void Put(Cache *c, Callback cb, Node *ptr, uptr size) { @@ -117,7 +116,16 @@ class Quarantine { cache_.Transfer(c); } if (cache_.Size() > GetSize() && recycle_mutex_.TryLock()) - Recycle(cb); + Recycle(atomic_load_relaxed(&min_size_), cb); + } + + void NOINLINE DrainAndRecycle(Cache *c, Callback cb) { + { + SpinMutexLock l(&cache_mutex_); + cache_.Transfer(c); + } + recycle_mutex_.Lock(); + Recycle(0, cb); } void PrintStats() const { @@ -139,9 +147,8 @@ class Quarantine { Cache cache_; char pad2_[kCacheLineSize]; - void NOINLINE Recycle(Callback cb) { + void NOINLINE Recycle(uptr min_size, Callback cb) { Cache tmp; - uptr min_size = atomic_load(&min_size_, memory_order_relaxed); { SpinMutexLock l(&cache_mutex_); // Go over the batches and merge partially filled ones to @@ -201,7 +208,7 @@ class QuarantineCache { // Total memory used, including internal accounting. uptr Size() const { - return atomic_load(&size_, memory_order_relaxed); + return atomic_load_relaxed(&size_); } // Memory used for internal accounting. @@ -225,7 +232,7 @@ class QuarantineCache { list_.append_back(&from_cache->list_); SizeAdd(from_cache->Size()); - atomic_store(&from_cache->size_, 0, memory_order_relaxed); + atomic_store_relaxed(&from_cache->size_, 0); } void EnqueueBatch(QuarantineBatch *b) { @@ -296,10 +303,10 @@ class QuarantineCache { atomic_uintptr_t size_; void SizeAdd(uptr add) { - atomic_store(&size_, Size() + add, memory_order_relaxed); + atomic_store_relaxed(&size_, Size() + add); } void SizeSub(uptr sub) { - atomic_store(&size_, Size() - sub, memory_order_relaxed); + atomic_store_relaxed(&size_, Size() - sub); } }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h index 86536aa19658..060b58d3f2d3 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h @@ -27,8 +27,9 @@ class SanitizerCommonDecorator { SanitizerCommonDecorator() : ansi_(ColorizeReports()) {} const char *Bold() const { return ansi_ ? "\033[1m" : ""; } const char *Default() const { return ansi_ ? "\033[1m\033[0m" : ""; } - const char *Warning() { return Red(); } - const char *EndWarning() { return Default(); } + const char *Warning() const { return Red(); } + const char *MemoryByte() { return Magenta(); } + protected: const char *Black() const { return ansi_ ? "\033[1m\033[30m" : ""; } const char *Red() const { return ansi_ ? "\033[1m\033[31m" : ""; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc new file mode 100644 index 000000000000..f51fc049e327 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc @@ -0,0 +1,87 @@ +//===-- sanitizer_signal_interceptors.inc -----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Signal interceptors for sanitizers. +// +//===----------------------------------------------------------------------===// + +#include "interception/interception.h" +#include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_interceptors.h" + +using namespace __sanitizer; + +#if SANITIZER_NETBSD +#define sigaction_symname __sigaction14 +#else +#define sigaction_symname sigaction +#endif + +#ifndef SIGNAL_INTERCEPTOR_SIGNAL_IMPL +#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signum, handler) \ + { return REAL(func)(signum, handler); } +#endif + +#ifndef SIGNAL_INTERCEPTOR_SIGACTION_IMPL +#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \ + { return REAL(sigaction_symname)(signum, act, oldact); } +#endif + +#if SANITIZER_INTERCEPT_BSD_SIGNAL +INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) { + if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0; + SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler); +} +#define INIT_BSD_SIGNAL COMMON_INTERCEPT_FUNCTION(bsd_signal) +#else // SANITIZER_INTERCEPT_BSD_SIGNAL +#define INIT_BSD_SIGNAL +#endif // SANITIZER_INTERCEPT_BSD_SIGNAL + +#if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION +INTERCEPTOR(uptr, signal, int signum, uptr handler) { + if (GetHandleSignalMode(signum) == kHandleSignalExclusive) + return (uptr) nullptr; + SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler); +} +#define INIT_SIGNAL COMMON_INTERCEPT_FUNCTION(signal) + +INTERCEPTOR(int, sigaction_symname, int signum, + const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) { + if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0; + SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact); +} +#define INIT_SIGACTION COMMON_INTERCEPT_FUNCTION(sigaction_symname) + +namespace __sanitizer { +int real_sigaction(int signum, const void *act, void *oldact) { + return REAL(sigaction_symname)(signum, (const __sanitizer_sigaction *)act, + (__sanitizer_sigaction *)oldact); +} +} // namespace __sanitizer +#else // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION +#define INIT_SIGNAL +#define INIT_SIGACTION +// We need to have defined REAL(sigaction) on other systems. +namespace __sanitizer { +struct __sanitizer_sigaction; +} +DEFINE_REAL(int, sigaction, int signum, const __sanitizer_sigaction *act, + __sanitizer_sigaction *oldact) +#endif // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION + +static void InitializeSignalInterceptors() { + static bool was_called_once; + CHECK(!was_called_once); + was_called_once = true; + + INIT_BSD_SIGNAL; + INIT_SIGNAL; + INIT_SIGACTION; +} diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc new file mode 100644 index 000000000000..a5db22994e0a --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc @@ -0,0 +1,219 @@ +//===-- sanitizer_solaris.cc ----------------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between various sanitizers' runtime libraries and +// implements Solaris-specific functions. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_SOLARIS + +#include <stdio.h> + +#include "sanitizer_common.h" +#include "sanitizer_flags.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_libc.h" +#include "sanitizer_placement_new.h" +#include "sanitizer_platform_limits_posix.h" +#include "sanitizer_procmaps.h" + +#include <fcntl.h> +#include <pthread.h> +#include <sched.h> +#include <thread.h> +#include <synch.h> +#include <signal.h> +#include <sys/mman.h> +#include <sys/resource.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <dirent.h> +#include <unistd.h> +#include <errno.h> +#include <stdlib.h> + +namespace __sanitizer { + +//#include "sanitizer_syscall_generic.inc" + +#define _REAL(func) _ ## func +#define DECLARE__REAL(ret_type, func, ...) \ + extern "C" ret_type _REAL(func)(__VA_ARGS__) +#define DECLARE__REAL_AND_INTERNAL(ret_type, func, ...) \ + DECLARE__REAL(ret_type, func, __VA_ARGS__); \ + ret_type internal_ ## func(__VA_ARGS__) + +// ---------------------- sanitizer_libc.h +DECLARE__REAL_AND_INTERNAL(uptr, mmap, void *addr, uptr /*size_t*/ length, + int prot, int flags, int fd, OFF_T offset) { + return (uptr)_REAL(mmap)(addr, length, prot, flags, fd, offset); +} + +DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) { + return _REAL(munmap)(addr, length); +} + +DECLARE__REAL_AND_INTERNAL(int, mprotect, void *addr, uptr length, int prot) { + return _REAL(mprotect)(addr, length, prot); +} + +DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) { + return _REAL(close)(fd); +} + +extern "C" int _REAL(open)(const char *, int, ...); + +uptr internal_open(const char *filename, int flags) { + return _REAL(open)(filename, flags); +} + +uptr internal_open(const char *filename, int flags, u32 mode) { + return _REAL(open)(filename, flags, mode); +} + +uptr OpenFile(const char *filename, bool write) { + return internal_open(filename, + write ? O_WRONLY | O_CREAT : O_RDONLY, 0660); +} + +DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) { + return _REAL(read)(fd, buf, count); +} + +DECLARE__REAL_AND_INTERNAL(uptr, write, fd_t fd, const void *buf, uptr count) { + return _REAL(write)(fd, buf, count); +} + +// FIXME: There's only _ftruncate64 beginning with Solaris 11. +DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) { + return ftruncate(fd, size); +} + +DECLARE__REAL_AND_INTERNAL(uptr, stat, const char *path, void *buf) { + return _REAL(stat)(path, (struct stat *)buf); +} + +DECLARE__REAL_AND_INTERNAL(uptr, lstat, const char *path, void *buf) { + return _REAL(lstat)(path, (struct stat *)buf); +} + +DECLARE__REAL_AND_INTERNAL(uptr, fstat, fd_t fd, void *buf) { + return _REAL(fstat)(fd, (struct stat *)buf); +} + +uptr internal_filesize(fd_t fd) { + struct stat st; + if (internal_fstat(fd, &st)) + return -1; + return (uptr)st.st_size; +} + +DECLARE__REAL_AND_INTERNAL(uptr, dup2, int oldfd, int newfd) { + return _REAL(dup2)(oldfd, newfd); +} + +DECLARE__REAL_AND_INTERNAL(uptr, readlink, const char *path, char *buf, + uptr bufsize) { + return _REAL(readlink)(path, buf, bufsize); +} + +DECLARE__REAL_AND_INTERNAL(uptr, unlink, const char *path) { + return _REAL(unlink)(path); +} + +DECLARE__REAL_AND_INTERNAL(uptr, rename, const char *oldpath, + const char *newpath) { + return _REAL(rename)(oldpath, newpath); +} + +DECLARE__REAL_AND_INTERNAL(uptr, sched_yield, void) { + return sched_yield(); +} + +DECLARE__REAL_AND_INTERNAL(void, _exit, int exitcode) { + _exit(exitcode); +} + +DECLARE__REAL_AND_INTERNAL(uptr, execve, const char *filename, + char *const argv[], char *const envp[]) { + return _REAL(execve)(filename, argv, envp); +} + +DECLARE__REAL_AND_INTERNAL(uptr, waitpid, int pid, int *status, int options) { + return _REAL(waitpid)(pid, status, options); +} + +DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) { + return _REAL(getpid)(); +} + +// FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *. +DECLARE__REAL_AND_INTERNAL(uptr, getdents, fd_t fd, struct linux_dirent *dirp, + unsigned int count) { + return _REAL(getdents)(fd, dirp, count); +} + +DECLARE__REAL_AND_INTERNAL(uptr, lseek, fd_t fd, OFF_T offset, int whence) { + return _REAL(lseek)(fd, offset, whence); +} + +// FIXME: This might be wrong: _sigfillset doesn't take a +// __sanitizer_sigset_t *. +DECLARE__REAL_AND_INTERNAL(void, sigfillset, __sanitizer_sigset_t *set) { + _REAL(sigfillset)(set); +} + +// FIXME: This might be wrong: _sigprocmask doesn't take __sanitizer_sigset_t *. +DECLARE__REAL_AND_INTERNAL(uptr, sigprocmask, int how, + __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + return _REAL(sigprocmask)(how, set, oldset); +} + +DECLARE__REAL_AND_INTERNAL(int, fork, void) { + // TODO(glider): this may call user's pthread_atfork() handlers which is bad. + return _REAL(fork)(); +} + +u64 NanoTime() { + return gethrtime(); +} + +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { + // FIXME: No internal variant. + return clock_gettime(clk_id, (timespec *)tp); +} + +// ----------------- sanitizer_common.h +BlockingMutex::BlockingMutex() { + CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_)); + internal_memset(this, 0, sizeof(*this)); + CHECK_EQ(mutex_init((mutex_t *)&opaque_storage_, USYNC_THREAD, NULL), 0); +} + +void BlockingMutex::Lock() { + CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_)); + CHECK_NE(owner_, (uptr)thr_self()); + CHECK_EQ(mutex_lock((mutex_t *)&opaque_storage_), 0); + CHECK(!owner_); + owner_ = (uptr)thr_self(); +} + +void BlockingMutex::Unlock() { + CHECK(owner_ == (uptr)thr_self()); + owner_ = 0; + CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0); +} + +void BlockingMutex::CheckLocked() { + CHECK_EQ((uptr)thr_self(), owner_); +} + +} // namespace __sanitizer + +#endif // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h index 90142dffd5e4..f15196e4b725 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h @@ -97,6 +97,11 @@ struct BufferedStackTrace : public StackTrace { void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind); + void Reset() { + *static_cast<StackTrace *>(this) = StackTrace(trace_buffer, 0); + top_frame_bp = 0; + } + private: void FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, u32 max_depth); @@ -106,8 +111,8 @@ struct BufferedStackTrace : public StackTrace { void PopStackFrames(uptr count); uptr LocatePcInTrace(uptr pc); - BufferedStackTrace(const BufferedStackTrace &); - void operator=(const BufferedStackTrace &); + BufferedStackTrace(const BufferedStackTrace &) = delete; + void operator=(const BufferedStackTrace &) = delete; }; // Check if given pointer points into allocated stack area. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc index 377f1ce755be..a271302708be 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -12,9 +12,14 @@ //===----------------------------------------------------------------------===// #include "sanitizer_stacktrace_printer.h" +#include "sanitizer_file.h" +#include "sanitizer_fuchsia.h" namespace __sanitizer { +// sanitizer_symbolizer_fuchsia.cc implements these differently for Fuchsia. +#if !SANITIZER_FUCHSIA + static const char *StripFunctionName(const char *function, const char *prefix) { if (!function) return nullptr; if (!prefix) return function; @@ -24,6 +29,80 @@ static const char *StripFunctionName(const char *function, const char *prefix) { return function; } +static const char *DemangleFunctionName(const char *function) { + if (!function) return nullptr; + + // NetBSD uses indirection for old threading functions for historical reasons + // The mangled names are internal implementation detail and should not be + // exposed even in backtraces. +#if SANITIZER_NETBSD + if (!internal_strcmp(function, "__libc_mutex_init")) + return "pthread_mutex_init"; + if (!internal_strcmp(function, "__libc_mutex_lock")) + return "pthread_mutex_lock"; + if (!internal_strcmp(function, "__libc_mutex_trylock")) + return "pthread_mutex_trylock"; + if (!internal_strcmp(function, "__libc_mutex_unlock")) + return "pthread_mutex_unlock"; + if (!internal_strcmp(function, "__libc_mutex_destroy")) + return "pthread_mutex_destroy"; + if (!internal_strcmp(function, "__libc_mutexattr_init")) + return "pthread_mutexattr_init"; + if (!internal_strcmp(function, "__libc_mutexattr_settype")) + return "pthread_mutexattr_settype"; + if (!internal_strcmp(function, "__libc_mutexattr_destroy")) + return "pthread_mutexattr_destroy"; + if (!internal_strcmp(function, "__libc_cond_init")) + return "pthread_cond_init"; + if (!internal_strcmp(function, "__libc_cond_signal")) + return "pthread_cond_signal"; + if (!internal_strcmp(function, "__libc_cond_broadcast")) + return "pthread_cond_broadcast"; + if (!internal_strcmp(function, "__libc_cond_wait")) + return "pthread_cond_wait"; + if (!internal_strcmp(function, "__libc_cond_timedwait")) + return "pthread_cond_timedwait"; + if (!internal_strcmp(function, "__libc_cond_destroy")) + return "pthread_cond_destroy"; + if (!internal_strcmp(function, "__libc_rwlock_init")) + return "pthread_rwlock_init"; + if (!internal_strcmp(function, "__libc_rwlock_rdlock")) + return "pthread_rwlock_rdlock"; + if (!internal_strcmp(function, "__libc_rwlock_wrlock")) + return "pthread_rwlock_wrlock"; + if (!internal_strcmp(function, "__libc_rwlock_tryrdlock")) + return "pthread_rwlock_tryrdlock"; + if (!internal_strcmp(function, "__libc_rwlock_trywrlock")) + return "pthread_rwlock_trywrlock"; + if (!internal_strcmp(function, "__libc_rwlock_unlock")) + return "pthread_rwlock_unlock"; + if (!internal_strcmp(function, "__libc_rwlock_destroy")) + return "pthread_rwlock_destroy"; + if (!internal_strcmp(function, "__libc_thr_keycreate")) + return "pthread_key_create"; + if (!internal_strcmp(function, "__libc_thr_setspecific")) + return "pthread_setspecific"; + if (!internal_strcmp(function, "__libc_thr_getspecific")) + return "pthread_getspecific"; + if (!internal_strcmp(function, "__libc_thr_keydelete")) + return "pthread_key_delete"; + if (!internal_strcmp(function, "__libc_thr_once")) + return "pthread_once"; + if (!internal_strcmp(function, "__libc_thr_self")) + return "pthread_self"; + if (!internal_strcmp(function, "__libc_thr_exit")) + return "pthread_exit"; + if (!internal_strcmp(function, "__libc_thr_setcancelstate")) + return "pthread_setcancelstate"; + if (!internal_strcmp(function, "__libc_thr_equal")) + return "pthread_equal"; + if (!internal_strcmp(function, "__libc_thr_curcpu")) + return "pthread_curcpu_np"; +#endif + + return function; +} + static const char kDefaultFormat[] = " #%n %p %F %L"; void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, @@ -55,7 +134,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, buffer->append("0x%zx", info.module_offset); break; case 'f': - buffer->append("%s", StripFunctionName(info.function, strip_func_prefix)); + buffer->append("%s", + DemangleFunctionName( + StripFunctionName(info.function, strip_func_prefix))); break; case 'q': buffer->append("0x%zx", info.function_offset != AddressInfo::kUnknown @@ -76,7 +157,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, // Function name and offset, if file is unknown. if (info.function) { buffer->append("in %s", - StripFunctionName(info.function, strip_func_prefix)); + DemangleFunctionName( + StripFunctionName(info.function, strip_func_prefix))); if (!info.file && info.function_offset != AddressInfo::kUnknown) buffer->append("+0x%zx", info.function_offset); } @@ -146,6 +228,8 @@ void RenderData(InternalScopedString *buffer, const char *format, } } +#endif // !SANITIZER_FUCHSIA + void RenderSourceLocation(InternalScopedString *buffer, const char *file, int line, int column, bool vs_style, const char *strip_path_prefix) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc index d7fa5f6451d1..0f543d2d7c87 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc @@ -57,6 +57,14 @@ #include "sanitizer_mutex.h" #include "sanitizer_placement_new.h" +// Sufficiently old kernel headers don't provide this value, but we can still +// call prctl with it. If the runtime kernel is new enough, the prctl call will +// have the desired effect; if the kernel is too old, the call will error and we +// can ignore said error. +#ifndef PR_SET_PTRACER +#define PR_SET_PTRACER 0x59616d61 +#endif + // This module works by spawning a Linux task which then attaches to every // thread in the caller process with ptrace. This suspends the threads, and // PTRACE_GETREGS can then be used to obtain their register state. The callback @@ -245,8 +253,9 @@ static void TracerThreadDieCallback() { } // Signal handler to wake up suspended threads when the tracer thread dies. -static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) { - SignalContext ctx = SignalContext::Create(siginfo, uctx); +static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo, + void *uctx) { + SignalContext ctx(siginfo, uctx); Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum, ctx.addr, ctx.pc, ctx.sp); ThreadSuspender *inst = thread_suspender_instance; @@ -263,7 +272,7 @@ static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) { } // Size of alternative stack for signal handlers in the tracer thread. -static const int kHandlerStackSize = 4096; +static const int kHandlerStackSize = 8192; // This function will be run as a cloned task. static int TracerThread(void* argument) { @@ -433,9 +442,7 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) { ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid); // On some systems we have to explicitly declare that we want to be traced // by the tracer thread. -#ifdef PR_SET_PTRACER internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0); -#endif // Allow the tracer thread to start. tracer_thread_argument.mutex.Unlock(); // NOTE: errno is shared between this thread and the tracer thread. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc index 0c27c472f02e..d84ebef3b7f9 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc @@ -55,17 +55,9 @@ void RunThread(void *arg) { struct RunThreadArgs *run_args = (struct RunThreadArgs *)arg; SuspendedThreadsListMac suspended_threads_list; - mach_port_t task; - kern_return_t err = task_for_pid(mach_task_self(), internal_getpid(), &task); - if (err != KERN_SUCCESS) { - VReport(1, "Getting task from pid failed (errno %d).\n", err); - return; - } - thread_array_t threads; mach_msg_type_number_t num_threads; - - err = task_threads(task, &threads, &num_threads); + kern_return_t err = task_threads(mach_task_self(), &threads, &num_threads); if (err != KERN_SUCCESS) { VReport(1, "Failed to get threads for task (errno %d).\n", err); return; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc index f0f2c9c725a1..89ddfddd1108 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc @@ -16,6 +16,7 @@ #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_flags.h" +#include "sanitizer_file.h" #include "sanitizer_libc.h" #include "sanitizer_placement_new.h" @@ -50,6 +51,7 @@ void SuppressionContext::ParseFromFile(const char *filename) { if (filename[0] == '\0') return; +#if !SANITIZER_FUCHSIA // If we cannot find the file, check if its location is relative to // the location of the executable. InternalScopedString new_file_path(kMaxPathLength); @@ -58,6 +60,7 @@ void SuppressionContext::ParseFromFile(const char *filename) { new_file_path.size())) { filename = new_file_path.data(); } +#endif // !SANITIZER_FUCHSIA // Read the file. VPrintf(1, "%s: reading suppressions file at %s\n", diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc index 1cd5b6ee24c0..672c936684be 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc @@ -71,6 +71,10 @@ Symbolizer *Symbolizer::symbolizer_; StaticSpinMutex Symbolizer::init_mu_; LowLevelAllocator Symbolizer::symbolizer_allocator_; +void Symbolizer::InvalidateModuleList() { + modules_fresh_ = false; +} + void Symbolizer::AddHooks(Symbolizer::StartSymbolizationHook start_hook, Symbolizer::EndSymbolizationHook end_hook) { CHECK(start_hook_ == 0 && end_hook_ == 0); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h index 4fc7742a21e3..208362d2f478 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h @@ -119,8 +119,11 @@ class Symbolizer final { void AddHooks(StartSymbolizationHook start_hook, EndSymbolizationHook end_hook); + void RefreshModules(); const LoadedModule *FindModuleForAddress(uptr address); + void InvalidateModuleList(); + private: // GetModuleNameAndOffsetForPC has to return a string to the caller. // Since the corresponding module might get unloaded later, we should create @@ -149,6 +152,7 @@ class Symbolizer final { uptr *module_offset, ModuleArch *module_arch); ListOfModules modules_; + ListOfModules fallback_modules_; // If stale, need to reload the modules before looking up addresses. bool modules_fresh_; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc new file mode 100644 index 000000000000..3d1117d9d3ca --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc @@ -0,0 +1,107 @@ +//===-- sanitizer_symbolizer_fuchsia.cc -----------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between various sanitizers' runtime libraries. +// +// Implementation of Fuchsia-specific symbolizer. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_FUCHSIA + +#include "sanitizer_fuchsia.h" +#include "sanitizer_symbolizer.h" + +namespace __sanitizer { + +// For Fuchsia we don't do any actual symbolization per se. +// Instead, we emit text containing raw addresses and raw linkage +// symbol names, embedded in Fuchsia's symbolization markup format. +// Fuchsia's logging infrastructure emits enough information about +// process memory layout that a post-processing filter can do the +// symbolization and pretty-print the markup. See the spec at: +// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md + +// This is used by UBSan for type names, and by ASan for global variable names. +constexpr const char *kFormatDemangle = "{{{symbol:%s}}}"; +constexpr uptr kFormatDemangleMax = 1024; // Arbitrary. + +// Function name or equivalent from PC location. +constexpr const char *kFormatFunction = "{{{pc:%p}}}"; +constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex. + +// Global variable name or equivalent from data memory address. +constexpr const char *kFormatData = "{{{data:%p}}}"; + +// One frame in a backtrace (printed on a line by itself). +constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}"; + +// This is used by UBSan for type names, and by ASan for global variable names. +// It's expected to return a static buffer that will be reused on each call. +const char *Symbolizer::Demangle(const char *name) { + static char buffer[kFormatDemangleMax]; + internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name); + return buffer; +} + +// This is used mostly for suppression matching. Making it work +// would enable "interceptor_via_lib" suppressions. It's also used +// once in UBSan to say "in module ..." in a message that also +// includes an address in the module, so post-processing can already +// pretty-print that so as to indicate the module. +bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, + uptr *module_address) { + return false; +} + +// This is used in some places for suppression checking, which we +// don't really support for Fuchsia. It's also used in UBSan to +// identify a PC location to a function name, so we always fill in +// the function member with a string containing markup around the PC +// value. +// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan +// to render stack frames, but that should be changed to use +// RenderStackFrame. +SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) { + SymbolizedStack *s = SymbolizedStack::New(addr); + char buffer[kFormatFunctionMax]; + internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr); + s->info.function = internal_strdup(buffer); + return s; +} + +// Always claim we succeeded, so that RenderDataInfo will be called. +bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) { + info->Clear(); + info->start = addr; + return true; +} + +// We ignore the format argument to __sanitizer_symbolize_global. +void RenderData(InternalScopedString *buffer, const char *format, + const DataInfo *DI, const char *strip_path_prefix) { + buffer->append(kFormatData, DI->start); +} + +// We don't support the stack_trace_format flag at all. +void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, + const AddressInfo &info, bool vs_style, + const char *strip_path_prefix, const char *strip_func_prefix) { + buffer->append(kFormatFrame, frame_no, info.address); +} + +Symbolizer *Symbolizer::PlatformInit() { + return new (symbolizer_allocator_) Symbolizer({}); +} + +void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); } + +} // namespace __sanitizer + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h index 2ae42b33868b..a2ac1188241d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h @@ -15,6 +15,7 @@ #define SANITIZER_SYMBOLIZER_INTERNAL_H #include "sanitizer_symbolizer.h" +#include "sanitizer_file.h" namespace __sanitizer { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc index 57354668bfbb..e98fab01d6bb 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc @@ -95,7 +95,8 @@ struct SymbolizeCodeCallbackArg { if (frames_symbolized > 0) { SymbolizedStack *cur = SymbolizedStack::New(addr); AddressInfo *info = &cur->info; - info->FillModuleInfo(first->info.module, first->info.module_offset); + info->FillModuleInfo(first->info.module, first->info.module_offset, + first->info.module_arch); last->next = cur; last = cur; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc index 614470a633d0..a4bab668b27a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc @@ -17,6 +17,18 @@ namespace __sanitizer { +Symbolizer *Symbolizer::GetOrInit() { + SpinMutexLock l(&init_mu_); + if (symbolizer_) + return symbolizer_; + symbolizer_ = PlatformInit(); + CHECK(symbolizer_); + return symbolizer_; +} + +// See sanitizer_symbolizer_fuchsia.cc. +#if !SANITIZER_FUCHSIA + const char *ExtractToken(const char *str, const char *delims, char **result) { uptr prefix_len = internal_strcspn(str, delims); *result = (char*)InternalAlloc(prefix_len + 1); @@ -151,37 +163,47 @@ bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address, return true; } +void Symbolizer::RefreshModules() { + modules_.init(); + fallback_modules_.fallbackInit(); + RAW_CHECK(modules_.size() > 0); + modules_fresh_ = true; +} + +static const LoadedModule *SearchForModule(const ListOfModules &modules, + uptr address) { + for (uptr i = 0; i < modules.size(); i++) { + if (modules[i].containsAddress(address)) { + return &modules[i]; + } + } + return nullptr; +} + const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) { bool modules_were_reloaded = false; if (!modules_fresh_) { - modules_.init(); - RAW_CHECK(modules_.size() > 0); - modules_fresh_ = true; + RefreshModules(); modules_were_reloaded = true; } - for (uptr i = 0; i < modules_.size(); i++) { - if (modules_[i].containsAddress(address)) { - return &modules_[i]; - } - } - // Reload the modules and look up again, if we haven't tried it yet. + const LoadedModule *module = SearchForModule(modules_, address); + if (module) return module; + + // dlopen/dlclose interceptors invalidate the module list, but when + // interception is disabled, we need to retry if the lookup fails in + // case the module list changed. +#if !SANITIZER_INTERCEPT_DLOPEN_DLCLOSE if (!modules_were_reloaded) { - // FIXME: set modules_fresh_ from dlopen()/dlclose() interceptors. - // It's too aggressive to reload the list of modules each time we fail - // to find a module for a given address. - modules_fresh_ = false; - return FindModuleForAddress(address); + RefreshModules(); + module = SearchForModule(modules_, address); + if (module) return module; } - return 0; -} +#endif -Symbolizer *Symbolizer::GetOrInit() { - SpinMutexLock l(&init_mu_); - if (symbolizer_) - return symbolizer_; - symbolizer_ = PlatformInit(); - CHECK(symbolizer_); - return symbolizer_; + if (fallback_modules_.size()) { + module = SearchForModule(fallback_modules_, address); + } + return module; } // For now we assume the following protocol: @@ -451,7 +473,7 @@ bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) { if (ReachedEndOfOutput(buffer, read_len)) break; if (read_len + 1 == max_length) { - Report("WARNING: Symbolizer buffer too small"); + Report("WARNING: Symbolizer buffer too small\n"); read_len = 0; break; } @@ -472,4 +494,6 @@ bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) { return true; } +#endif // !SANITIZER_FUCHSIA + } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc index 60ec7506c312..71d748d0515f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc @@ -16,6 +16,7 @@ #if SANITIZER_POSIX #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_internal_defs.h" #include "sanitizer_linux.h" @@ -76,6 +77,7 @@ static swift_demangle_ft swift_demangle_f; // symbolication. static void InitializeSwiftDemangler() { swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle"); + (void)dlerror(); // Cleanup error message in case of failure } // Attempts to demangle a Swift name. The demangler will return nullptr if a @@ -271,6 +273,10 @@ class Addr2LineProcess : public SymbolizerProcess { bool ReadFromSymbolizer(char *buffer, uptr max_length) override { if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length)) return false; + // The returned buffer is empty when output is valid, but exceeds + // max_length. + if (*buffer == '\0') + return true; // We should cut out output_terminator_ at the end of given buffer, // appended by addr2line to mark the end of its meaningful output. // We cannot scan buffer from it's beginning, because it is legal for it @@ -470,16 +476,16 @@ static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) { // Otherwise symbolizer program is unknown, let's search $PATH CHECK(path == nullptr); - if (const char *found_path = FindPathToBinary("llvm-symbolizer")) { - VReport(2, "Using llvm-symbolizer found at: %s\n", found_path); - return new(*allocator) LLVMSymbolizer(found_path, allocator); - } #if SANITIZER_MAC if (const char *found_path = FindPathToBinary("atos")) { VReport(2, "Using atos found at: %s\n", found_path); return new(*allocator) AtosSymbolizer(found_path, allocator); } #endif // SANITIZER_MAC + if (const char *found_path = FindPathToBinary("llvm-symbolizer")) { + VReport(2, "Using llvm-symbolizer found at: %s\n", found_path); + return new(*allocator) LLVMSymbolizer(found_path, allocator); + } if (common_flags()->allow_addr2line) { if (const char *found_path = FindPathToBinary("addr2line")) { VReport(2, "Using addr2line found at: %s\n", found_path); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc index 15cf05f06087..3c8e77867ce5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc @@ -7,20 +7,49 @@ // //===----------------------------------------------------------------------===// // -// Generic implementations of internal_syscall and internal_iserror. +// Generic implementations of internal_syscall* and internal_iserror. // //===----------------------------------------------------------------------===// -#if SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS # define SYSCALL(name) SYS_ ## name #else # define SYSCALL(name) __NR_ ## name #endif -#if (SANITIZER_FREEBSD || SANITIZER_MAC) && defined(__x86_64__) +#if SANITIZER_NETBSD +// We use 3 kinds of internal_syscall's for different types of retval in order +// to address differences in calling conventions (e.g. registers to place the +// return value in). +// - internal_syscall for 32-bit length (int, pid_t) +// - internal_syscall64 for 64-bit length (off_t) +// - internal_syscall_ptr for pointer and (s)size_t +# define internal_syscall syscall +# define internal_syscall64 __syscall +// Handle syscall renames manually +# define SYS_stat SYS___stat50 +# define SYS_lstat SYS___lstat50 +# define SYS_fstat SYS___fstat50 +# define SYS_gettimeofday SYS___gettimeofday50 +# define SYS_wait4 SYS___wait450 +# define SYS_getdents SYS___getdents30 +# define SYS_sigaltstack SYS___sigaltstack14 +# define SYS_sigprocmask SYS___sigprocmask14 +# define SYS_nanosleep SYS___nanosleep50 +# define SYS_clock_gettime SYS___clock_gettime50 +# if SANITIZER_WORDSIZE == 64 +# define internal_syscall_ptr __syscall +# else +# define internal_syscall_ptr syscall +# endif +#elif defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC) # define internal_syscall __syscall +# define internal_syscall64 __syscall +# define internal_syscall_ptr __syscall # else # define internal_syscall syscall +# define internal_syscall64 syscall +# define internal_syscall_ptr syscall #endif bool internal_iserror(uptr retval, int *rverrno) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc index 7ab1d7641449..1f05ed9b6c1c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc @@ -127,6 +127,9 @@ static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, #define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) +#define internal_syscall_ptr internal_syscall +#define internal_syscall64 internal_syscall + // Helper function used to avoid cobbler errno. bool internal_iserror(uptr retval, int *rverrno) { if (retval >= (uptr)-4095) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc new file mode 100644 index 000000000000..a3fdb9e60d5d --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc @@ -0,0 +1,141 @@ +//===-- sanitizer_syscall_linux_arm.inc -------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementations of internal_syscall and internal_iserror for Linux/arm. +// +//===----------------------------------------------------------------------===// + +#define SYSCALL(name) __NR_ ## name + +static uptr __internal_syscall(u32 nr) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0"); + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8) + : "memory", "cc"); + return r0; +} +#define __internal_syscall0(n) \ + (__internal_syscall)(n) + +static uptr __internal_syscall(u32 nr, u32 arg1) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0) + : "memory", "cc"); + return r0; +} +#define __internal_syscall1(n, a1) \ + (__internal_syscall)(n, (u32)(a1)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1) + : "memory", "cc"); + return r0; +} +#define __internal_syscall2(n, a1, a2) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2) + : "memory", "cc"); + return r0; +} +#define __internal_syscall3(n, a1, a2, a3) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, + u32 arg4) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + register u32 r3 asm("r3") = arg4; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3) + : "memory", "cc"); + return r0; +} +#define __internal_syscall4(n, a1, a2, a3, a4) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, + u32 arg4, long arg5) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + register u32 r3 asm("r3") = arg4; + register u32 r4 asm("r4") = arg5; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4) + : "memory", "cc"); + return r0; +} +#define __internal_syscall5(n, a1, a2, a3, a4, a5) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u32)(a5)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, + u32 arg4, long arg5, long arg6) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + register u32 r3 asm("r3") = arg4; + register u32 r4 asm("r4") = arg5; + register u32 r5 asm("r5") = arg6; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5) + : "memory", "cc"); + return r0; +} +#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u32)(a5), (long)(a6)) + +#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n +#define __SYSCALL_NARGS(...) \ + __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) +#define __SYSCALL_CONCAT_X(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b) +#define __SYSCALL_DISP(b, ...) \ + __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) + +#define internal_syscall_ptr internal_syscall +#define internal_syscall64 internal_syscall + +// Helper function used to avoid cobbler errno. +bool internal_iserror(uptr retval, int *rverrno) { + if (retval >= (uptr)-4095) { + if (rverrno) + *rverrno = -retval; + return true; + } + return false; +} diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc index 9853a6a675d3..327aaa80a674 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc @@ -20,6 +20,9 @@ static uptr internal_syscall(u64 nr) { return retval; } +#define internal_syscall_ptr internal_syscall +#define internal_syscall64 internal_syscall + template <typename T1> static uptr internal_syscall(u64 nr, T1 arg1) { u64 retval; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc index 439e33a08d01..d9fd6549ba48 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc @@ -21,6 +21,7 @@ ThreadContextBase::ThreadContextBase(u32 tid) status(ThreadStatusInvalid), detached(false), workerthread(false), parent_tid(0), next(0) { name[0] = '\0'; + atomic_store(&thread_destroyed, 0, memory_order_release); } ThreadContextBase::~ThreadContextBase() { @@ -44,6 +45,14 @@ void ThreadContextBase::SetDead() { OnDead(); } +void ThreadContextBase::SetDestroyed() { + atomic_store(&thread_destroyed, 1, memory_order_release); +} + +bool ThreadContextBase::GetDestroyed() { + return !!atomic_load(&thread_destroyed, memory_order_acquire); +} + void ThreadContextBase::SetJoined(void *arg) { // FIXME(dvyukov): print message and continue (it's user error). CHECK_EQ(false, detached); @@ -54,8 +63,11 @@ void ThreadContextBase::SetJoined(void *arg) { } void ThreadContextBase::SetFinished() { - if (!detached) - status = ThreadStatusFinished; + // ThreadRegistry::FinishThread calls here in ThreadStatusCreated state + // for a thread that never actually started. In that case the thread + // should go to ThreadStatusFinished regardless of whether it was created + // as detached. + if (!detached || status == ThreadStatusCreated) status = ThreadStatusFinished; OnFinished(); } @@ -82,6 +94,7 @@ void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id, void ThreadContextBase::Reset() { status = ThreadStatusInvalid; SetName(0); + atomic_store(&thread_destroyed, 0, memory_order_release); OnReset(); } @@ -204,7 +217,8 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) { CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); - CHECK_EQ(ThreadStatusRunning, tctx->status); + CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning, + tctx->status); tctx->SetName(name); } @@ -239,33 +253,54 @@ void ThreadRegistry::DetachThread(u32 tid, void *arg) { } void ThreadRegistry::JoinThread(u32 tid, void *arg) { - BlockingMutexLock l(&mtx_); - CHECK_LT(tid, n_contexts_); - ThreadContextBase *tctx = threads_[tid]; - CHECK_NE(tctx, 0); - if (tctx->status == ThreadStatusInvalid) { - Report("%s: Join of non-existent thread\n", SanitizerToolName); - return; - } - tctx->SetJoined(arg); - QuarantinePush(tctx); + bool destroyed = false; + do { + { + BlockingMutexLock l(&mtx_); + CHECK_LT(tid, n_contexts_); + ThreadContextBase *tctx = threads_[tid]; + CHECK_NE(tctx, 0); + if (tctx->status == ThreadStatusInvalid) { + Report("%s: Join of non-existent thread\n", SanitizerToolName); + return; + } + if ((destroyed = tctx->GetDestroyed())) { + tctx->SetJoined(arg); + QuarantinePush(tctx); + } + } + if (!destroyed) + internal_sched_yield(); + } while (!destroyed); } +// Normally this is called when the thread is about to exit. If +// called in ThreadStatusCreated state, then this thread was never +// really started. We just did CreateThread for a prospective new +// thread before trying to create it, and then failed to actually +// create it, and so never called StartThread. void ThreadRegistry::FinishThread(u32 tid) { BlockingMutexLock l(&mtx_); CHECK_GT(alive_threads_, 0); alive_threads_--; - CHECK_GT(running_threads_, 0); - running_threads_--; CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); - CHECK_EQ(ThreadStatusRunning, tctx->status); + bool dead = tctx->detached; + if (tctx->status == ThreadStatusRunning) { + CHECK_GT(running_threads_, 0); + running_threads_--; + } else { + // The thread never really existed. + CHECK_EQ(tctx->status, ThreadStatusCreated); + dead = true; + } tctx->SetFinished(); - if (tctx->detached) { + if (dead) { tctx->SetDead(); QuarantinePush(tctx); } + tctx->SetDestroyed(); } void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread, diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h index 9aae875c7360..b203be2f4dbc 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h @@ -50,6 +50,8 @@ class ThreadContextBase { u32 parent_tid; ThreadContextBase *next; // For storing thread contexts in a list. + atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished + void SetName(const char *new_name); void SetDead(); @@ -60,6 +62,9 @@ class ThreadContextBase { u32 _parent_tid, void *arg); void Reset(); + void SetDestroyed(); + bool GetDestroyed(); + // The following methods may be overriden by subclasses. // Some of them take opaque arg that may be optionally be used // by subclasses. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc index 29db37b8a464..6fa0e9298b31 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc @@ -45,7 +45,7 @@ static const uptr kDestroyedThread = -1; static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) { if (!size) return; - VPrintf(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size); + VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size); UnmapOrDie(dtv, size * sizeof(DTLS::DTV)); atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed); } @@ -58,7 +58,7 @@ static inline void DTLS_Resize(uptr new_size) { (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize"); uptr num_live_dtls = atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed); - VPrintf(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls); + VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls); CHECK_LT(num_live_dtls, 1 << 20); uptr old_dtv_size = dtls.dtv_size; DTLS::DTV *old_dtv = dtls.dtv; @@ -72,7 +72,7 @@ static inline void DTLS_Resize(uptr new_size) { void DTLS_Destroy() { if (!common_flags()->intercept_tls_get_addr) return; - VPrintf(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size); + VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size); uptr s = dtls.dtv_size; dtls.dtv_size = kDestroyedThread; // Do this before unmap for AS-safety. DTLS_Deallocate(dtls.dtv, s); @@ -97,28 +97,28 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, if (dtls.dtv[dso_id].beg) return 0; uptr tls_size = 0; uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset; - VPrintf(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p " + VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p " "num_live_dtls %zd\n", arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg, atomic_load(&number_of_live_dtls, memory_order_relaxed)); if (dtls.last_memalign_ptr == tls_beg) { tls_size = dtls.last_memalign_size; - VPrintf(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n", + VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n", tls_beg, tls_size); } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) { // This is the static TLS block which was initialized / unpoisoned at thread // creation. - VPrintf(2, "__tls_get_addr: static tls: %p\n", tls_beg); + VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg); tls_size = 0; } else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) { // We may want to check gnu_get_libc_version(). Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1; tls_size = header->size; tls_beg = header->start; - VPrintf(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n", + VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n", tls_beg, tls_size); } else { - VPrintf(2, "__tls_get_addr: Can't guess glibc version\n"); + VReport(2, "__tls_get_addr: Can't guess glibc version\n"); // This may happen inside the DTOR of main thread, so just ignore it. tls_size = 0; } @@ -129,7 +129,7 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, void DTLS_on_libc_memalign(void *ptr, uptr size) { if (!common_flags()->intercept_tls_get_addr) return; - VPrintf(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size); + VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size); dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr); dtls.last_memalign_size = size; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc index 5943125c898c..24c94700173d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc @@ -8,11 +8,12 @@ //===----------------------------------------------------------------------===// // // This file contains the unwind.h-based (aka "slow") stack unwinding routines -// available to the tools on Linux, Android, and FreeBSD. +// available to the tools on Linux, Android, NetBSD, FreeBSD, and Solaris. //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_stacktrace.h" @@ -78,7 +79,8 @@ void SanitizerInitializeUnwinder() { } #endif -#ifdef __arm__ +#if defined(__arm__) && !SANITIZER_NETBSD +// NetBSD uses dwarf EH #define UNWIND_STOP _URC_END_OF_STACK #define UNWIND_CONTINUE _URC_NO_REASON #else @@ -165,4 +167,5 @@ void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h index a7fb3fa58d54..25cfeed35f22 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h @@ -1,4 +1,4 @@ -//===-- tsan_vector.h -------------------------------------------*- C++ -*-===// +//===-- sanitizer_vector.h -------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,38 +7,37 @@ // //===----------------------------------------------------------------------===// // -// This file is a part of ThreadSanitizer (TSan), a race detector. +// This file is shared between sanitizers run-time libraries. // //===----------------------------------------------------------------------===// // Low-fat STL-like vector container. -#ifndef TSAN_VECTOR_H -#define TSAN_VECTOR_H +#ifndef SANITIZER_VECTOR_H +#define SANITIZER_VECTOR_H -#include "tsan_defs.h" -#include "tsan_mman.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_libc.h" -namespace __tsan { +namespace __sanitizer { template<typename T> class Vector { public: - explicit Vector(MBlockType typ) - : typ_(typ) - , begin_() + explicit Vector() + : begin_() , end_() , last_() { } ~Vector() { if (begin_) - internal_free(begin_); + InternalFree(begin_); } void Reset() { if (begin_) - internal_free(begin_); + InternalFree(begin_); begin_ = 0; end_ = 0; last_ = 0; @@ -91,7 +90,6 @@ class Vector { } private: - const MBlockType typ_; T *begin_; T *end_; T *last_; @@ -109,10 +107,10 @@ class Vector { cap = 16; if (cap < size) cap = size; - T *p = (T*)internal_alloc(typ_, cap * sizeof(T)); + T *p = (T*)InternalAlloc(cap * sizeof(T)); if (cap0) { internal_memcpy(p, begin_, cap0 * sizeof(T)); - internal_free(begin_); + InternalFree(begin_); } begin_ = p; end_ = begin_ + size; @@ -122,6 +120,6 @@ class Vector { Vector(const Vector&); void operator=(const Vector&); }; -} // namespace __tsan +} // namespace __sanitizer -#endif // #ifndef TSAN_VECTOR_H +#endif // #ifndef SANITIZER_VECTOR_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc index de01e8d119a1..34bf1812d605 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc @@ -24,10 +24,10 @@ #include "sanitizer_common.h" #include "sanitizer_dbghelp.h" +#include "sanitizer_file.h" #include "sanitizer_libc.h" #include "sanitizer_mutex.h" #include "sanitizer_placement_new.h" -#include "sanitizer_procmaps.h" #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" #include "sanitizer_win_defs.h" @@ -64,12 +64,16 @@ uptr GetMmapGranularity() { return si.dwAllocationGranularity; } -uptr GetMaxVirtualAddress() { +uptr GetMaxUserVirtualAddress() { SYSTEM_INFO si; GetSystemInfo(&si); return (uptr)si.lpMaximumApplicationAddress; } +uptr GetMaxVirtualAddress() { + return GetMaxUserVirtualAddress(); +} + bool FileExists(const char *filename) { return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES; } @@ -235,6 +239,28 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) { return p; } +// Uses fixed_addr for now. +// Will use offset instead once we've implemented this function for real. +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { + return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size)); +} + +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { + return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size)); +} + +void ReservedAddressRange::Unmap(uptr addr, uptr size) { + void* addr_as_void = reinterpret_cast<void*>(addr); + uptr base_as_uptr = reinterpret_cast<uptr>(base_); + // Only unmap if it covers the entire range. + CHECK((addr == base_as_uptr) && (size == size_)); + UnmapOrDie(addr_as_void, size); + if (addr_as_void == base_) { + base_ = reinterpret_cast<void*>(addr + size); + } + size_ = size_ - size; +} + void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) { void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_COMMIT, PAGE_READWRITE); @@ -252,6 +278,19 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) { return MmapOrDie(size, mem_type); } +uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { + if (fixed_addr) { + base_ = MmapFixedNoAccess(fixed_addr, size, name); + } else { + base_ = MmapNoAccess(size); + } + size_ = size; + name_ = name; + (void)os_handle_; // unsupported + return reinterpret_cast<uptr>(base_); +} + + void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { (void)name; // unsupported void *res = VirtualAlloc((LPVOID)fixed_addr, size, @@ -379,7 +418,7 @@ struct ModuleInfo { #if !SANITIZER_GO int CompareModulesBase(const void *pl, const void *pr) { - const ModuleInfo *l = (ModuleInfo *)pl, *r = (ModuleInfo *)pr; + const ModuleInfo *l = (const ModuleInfo *)pl, *r = (const ModuleInfo *)pr; if (l->base_address < r->base_address) return -1; return l->base_address > r->base_address; @@ -466,6 +505,10 @@ u64 NanoTime() { return 0; } +u64 MonotonicNanoTime() { + return 0; +} + void Abort() { internal__exit(3); } @@ -524,7 +567,7 @@ static uptr GetPreferredBase(const char *modname) { } void ListOfModules::init() { - clear(); + clearOrInit(); HANDLE cur_process = GetCurrentProcess(); // Query the list of modules. Start by assuming there are no more than 256 @@ -583,7 +626,9 @@ void ListOfModules::init() { modules_.push_back(cur_module); } UnmapOrDie(hmodules, modules_buffer_size); -}; +} + +void ListOfModules::fallbackInit() { clear(); } // We can't use atexit() directly at __asan_init time as the CRT is not fully // initialized at this point. Place the functions into a vector and use @@ -792,7 +837,7 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { // FIXME: Compare with StackWalk64. // FIXME: Look at LLVMUnhandledExceptionFilter in Signals.inc size = CaptureStackBackTrace(1, Min(max_depth, kStackTraceMax), - (void**)trace, 0); + (void **)&trace_buffer[0], 0); if (size == 0) return; @@ -889,32 +934,6 @@ bool IsHandledDeadlyException(DWORD exceptionCode) { return false; } -const char *DescribeSignalOrException(int signo) { - unsigned code = signo; - // Get the string description of the exception if this is a known deadly - // exception. - switch (code) { - case EXCEPTION_ACCESS_VIOLATION: return "access-violation"; - case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: return "array-bounds-exceeded"; - case EXCEPTION_STACK_OVERFLOW: return "stack-overflow"; - case EXCEPTION_DATATYPE_MISALIGNMENT: return "datatype-misalignment"; - case EXCEPTION_IN_PAGE_ERROR: return "in-page-error"; - case EXCEPTION_ILLEGAL_INSTRUCTION: return "illegal-instruction"; - case EXCEPTION_PRIV_INSTRUCTION: return "priv-instruction"; - case EXCEPTION_BREAKPOINT: return "breakpoint"; - case EXCEPTION_FLT_DENORMAL_OPERAND: return "flt-denormal-operand"; - case EXCEPTION_FLT_DIVIDE_BY_ZERO: return "flt-divide-by-zero"; - case EXCEPTION_FLT_INEXACT_RESULT: return "flt-inexact-result"; - case EXCEPTION_FLT_INVALID_OPERATION: return "flt-invalid-operation"; - case EXCEPTION_FLT_OVERFLOW: return "flt-overflow"; - case EXCEPTION_FLT_STACK_CHECK: return "flt-stack-check"; - case EXCEPTION_FLT_UNDERFLOW: return "flt-underflow"; - case EXCEPTION_INT_DIVIDE_BY_ZERO: return "int-divide-by-zero"; - case EXCEPTION_INT_OVERFLOW: return "int-overflow"; - } - return "unknown exception"; -} - bool IsAccessibleMemoryRange(uptr beg, uptr size) { SYSTEM_INFO si; GetNativeSystemInfo(&si); @@ -940,39 +959,101 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) { return true; } -SignalContext SignalContext::Create(void *siginfo, void *context) { +bool SignalContext::IsStackOverflow() const { + return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW; +} + +void SignalContext::InitPcSpBp() { EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo; CONTEXT *context_record = (CONTEXT *)context; - uptr pc = (uptr)exception_record->ExceptionAddress; + pc = (uptr)exception_record->ExceptionAddress; #ifdef _WIN64 - uptr bp = (uptr)context_record->Rbp; - uptr sp = (uptr)context_record->Rsp; + bp = (uptr)context_record->Rbp; + sp = (uptr)context_record->Rsp; #else - uptr bp = (uptr)context_record->Ebp; - uptr sp = (uptr)context_record->Esp; + bp = (uptr)context_record->Ebp; + sp = (uptr)context_record->Esp; #endif - uptr access_addr = exception_record->ExceptionInformation[1]; +} + +uptr SignalContext::GetAddress() const { + EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo; + return exception_record->ExceptionInformation[1]; +} + +bool SignalContext::IsMemoryAccess() const { + return GetWriteFlag() != SignalContext::UNKNOWN; +} +SignalContext::WriteFlag SignalContext::GetWriteFlag() const { + EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo; // The contents of this array are documented at // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx // The first element indicates read as 0, write as 1, or execute as 8. The // second element is the faulting address. - WriteFlag write_flag = SignalContext::UNKNOWN; switch (exception_record->ExceptionInformation[0]) { - case 0: write_flag = SignalContext::READ; break; - case 1: write_flag = SignalContext::WRITE; break; - case 8: write_flag = SignalContext::UNKNOWN; break; + case 0: + return SignalContext::READ; + case 1: + return SignalContext::WRITE; + case 8: + return SignalContext::UNKNOWN; } - bool is_memory_access = write_flag != SignalContext::UNKNOWN; - return SignalContext(context, access_addr, pc, sp, bp, is_memory_access, - write_flag); + return SignalContext::UNKNOWN; } void SignalContext::DumpAllRegisters(void *context) { // FIXME: Implement this. } +int SignalContext::GetType() const { + return static_cast<const EXCEPTION_RECORD *>(siginfo)->ExceptionCode; +} + +const char *SignalContext::Describe() const { + unsigned code = GetType(); + // Get the string description of the exception if this is a known deadly + // exception. + switch (code) { + case EXCEPTION_ACCESS_VIOLATION: + return "access-violation"; + case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: + return "array-bounds-exceeded"; + case EXCEPTION_STACK_OVERFLOW: + return "stack-overflow"; + case EXCEPTION_DATATYPE_MISALIGNMENT: + return "datatype-misalignment"; + case EXCEPTION_IN_PAGE_ERROR: + return "in-page-error"; + case EXCEPTION_ILLEGAL_INSTRUCTION: + return "illegal-instruction"; + case EXCEPTION_PRIV_INSTRUCTION: + return "priv-instruction"; + case EXCEPTION_BREAKPOINT: + return "breakpoint"; + case EXCEPTION_FLT_DENORMAL_OPERAND: + return "flt-denormal-operand"; + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + return "flt-divide-by-zero"; + case EXCEPTION_FLT_INEXACT_RESULT: + return "flt-inexact-result"; + case EXCEPTION_FLT_INVALID_OPERATION: + return "flt-invalid-operation"; + case EXCEPTION_FLT_OVERFLOW: + return "flt-overflow"; + case EXCEPTION_FLT_STACK_CHECK: + return "flt-stack-check"; + case EXCEPTION_FLT_UNDERFLOW: + return "flt-underflow"; + case EXCEPTION_INT_DIVIDE_BY_ZERO: + return "int-divide-by-zero"; + case EXCEPTION_INT_OVERFLOW: + return "int-overflow"; + } + return "unknown exception"; +} + uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { // FIXME: Actually implement this function. CHECK_GT(buf_len, 0); @@ -1021,7 +1102,12 @@ void CheckNoDeepBind(const char *filename, int flag) { } // FIXME: implement on this platform. -bool GetRandom(void *buffer, uptr length) { +bool GetRandom(void *buffer, uptr length, bool blocking) { + UNIMPLEMENTED(); +} + +// FIXME: implement on this platform. +u32 GetNumberOfCPUs() { UNIMPLEMENTED(); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc index 364319398198..5711f5dc8b6a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc @@ -12,7 +12,7 @@ // definition is provided. //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" +#include "sanitizer_platform.h" #if SANITIZER_WINDOWS && SANITIZER_DYNAMIC #include "sanitizer_win_weak_interception.h" #include "sanitizer_allocator_interface.h" diff --git a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh index c5865ecfee6c..0559a2e7eb05 100755 --- a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh +++ b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh @@ -110,12 +110,12 @@ cd ${LIBCXX_BUILD} ninja cxx cxxabi FLAGS="${FLAGS} -fno-rtti -fno-exceptions" +LLVM_FLAGS="${FLAGS} -nostdinc++ -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1" # Build LLVM. if [[ ! -d ${LLVM_BUILD} ]]; then mkdir -p ${LLVM_BUILD} cd ${LLVM_BUILD} - LLVM_FLAGS="${FLAGS} -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1" cmake -GNinja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=$CC \ @@ -137,7 +137,7 @@ mkdir ${SYMBOLIZER_BUILD} cd ${SYMBOLIZER_BUILD} echo "Compiling..." -SYMBOLIZER_FLAGS="$FLAGS -std=c++11 -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -I${LIBCXX_BUILD}/include/c++/v1" +SYMBOLIZER_FLAGS="$LLVM_FLAGS -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -std=c++11" $CXX $SYMBOLIZER_FLAGS ${SRC_DIR}/sanitizer_symbolize.cc ${SRC_DIR}/sanitizer_wrappers.cc -c $AR rc symbolizer.a sanitizer_symbolize.o sanitizer_wrappers.o diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp index 6f30ee987513..e5a4d714c66e 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp @@ -16,26 +16,27 @@ #include "scudo_allocator.h" #include "scudo_crc32.h" -#include "scudo_tls.h" +#include "scudo_flags.h" +#include "scudo_tsd.h" #include "scudo_utils.h" #include "sanitizer_common/sanitizer_allocator_checks.h" #include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_quarantine.h" +#include <errno.h> #include <string.h> namespace __scudo { // Global static cookie, initialized at start-up. -static uptr Cookie; +static u32 Cookie; // We default to software CRC32 if the alternatives are not supported, either // at compilation or at runtime. static atomic_uint8_t HashAlgorithm = { CRC32Software }; -INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) { +INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) { // If the hardware CRC32 feature is defined here, it was enabled everywhere, // as opposed to only for scudo_crc32.cpp. This means that other hardware // specific instructions were likely emitted at other places, and as a @@ -61,46 +62,60 @@ INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) { static ScudoBackendAllocator &getBackendAllocator(); -struct ScudoChunk : UnpackedHeader { +namespace Chunk { // We can't use the offset member of the chunk itself, as we would double // fetch it without any warranty that it wouldn't have been tampered. To // prevent this, we work with a local copy of the header. - void *getAllocBeg(UnpackedHeader *Header) { - return reinterpret_cast<void *>( - reinterpret_cast<uptr>(this) - (Header->Offset << MinAlignmentLog)); + static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) { + return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) - + AlignedChunkHeaderSize - + (Header->Offset << MinAlignmentLog)); + } + + static INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) { + return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) - + AlignedChunkHeaderSize); + } + static INLINE + const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { + return reinterpret_cast<const AtomicPackedHeader *>( + reinterpret_cast<uptr>(Ptr) - AlignedChunkHeaderSize); + } + + static INLINE bool isAligned(const void *Ptr) { + return IsAligned(reinterpret_cast<uptr>(Ptr), MinAlignment); } // Returns the usable size for a chunk, meaning the amount of bytes from the // beginning of the user data to the end of the backend allocated chunk. - uptr getUsableSize(UnpackedHeader *Header) { - uptr Size = - getBackendAllocator().getActuallyAllocatedSize(getAllocBeg(Header), - Header->FromPrimary); + static INLINE uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) { + const uptr Size = getBackendAllocator().getActuallyAllocatedSize( + getBackendPtr(Ptr, Header), Header->ClassId); if (Size == 0) return 0; return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog); } - // Compute the checksum of the Chunk pointer and its ChunkHeader. - u16 computeChecksum(UnpackedHeader *Header) const { + // Compute the checksum of the chunk pointer and its header. + static INLINE u16 computeChecksum(const void *Ptr, UnpackedHeader *Header) { UnpackedHeader ZeroChecksumHeader = *Header; ZeroChecksumHeader.Checksum = 0; uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); - u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(this), HeaderHolder, - ARRAY_SIZE(HeaderHolder)); + const u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(Ptr), + HeaderHolder, ARRAY_SIZE(HeaderHolder)); return static_cast<u16>(Crc); } // Checks the validity of a chunk by verifying its checksum. It doesn't // incur termination in the event of an invalid chunk. - bool isValid() { - UnpackedHeader NewUnpackedHeader; - const AtomicPackedHeader *AtomicHeader = - reinterpret_cast<const AtomicPackedHeader *>(this); - PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader); - NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader); - return (NewUnpackedHeader.Checksum == computeChecksum(&NewUnpackedHeader)); + static INLINE bool isValid(const void *Ptr) { + PackedHeader NewPackedHeader = + atomic_load_relaxed(getConstAtomicHeader(Ptr)); + UnpackedHeader NewUnpackedHeader = + bit_cast<UnpackedHeader>(NewPackedHeader); + return (NewUnpackedHeader.Checksum == + computeChecksum(Ptr, &NewUnpackedHeader)); } // Nulls out a chunk header. When returning the chunk to the backend, there @@ -109,114 +124,46 @@ struct ScudoChunk : UnpackedHeader { // the header invalid. In the extremely rare event where 0 would be a valid // checksum for the chunk, the state of the chunk is ChunkAvailable anyway. COMPILER_CHECK(ChunkAvailable == 0); - void eraseHeader() { - PackedHeader NullPackedHeader = 0; - AtomicPackedHeader *AtomicHeader = - reinterpret_cast<AtomicPackedHeader *>(this); - atomic_store_relaxed(AtomicHeader, NullPackedHeader); + static INLINE void eraseHeader(void *Ptr) { + const PackedHeader NullPackedHeader = 0; + atomic_store_relaxed(getAtomicHeader(Ptr), NullPackedHeader); } // Loads and unpacks the header, verifying the checksum in the process. - void loadHeader(UnpackedHeader *NewUnpackedHeader) const { - const AtomicPackedHeader *AtomicHeader = - reinterpret_cast<const AtomicPackedHeader *>(this); - PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader); + static INLINE + void loadHeader(const void *Ptr, UnpackedHeader *NewUnpackedHeader) { + PackedHeader NewPackedHeader = + atomic_load_relaxed(getConstAtomicHeader(Ptr)); *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader); if (UNLIKELY(NewUnpackedHeader->Checksum != - computeChecksum(NewUnpackedHeader))) { - dieWithMessage("ERROR: corrupted chunk header at address %p\n", this); + computeChecksum(Ptr, NewUnpackedHeader))) { + dieWithMessage("ERROR: corrupted chunk header at address %p\n", Ptr); } } // Packs and stores the header, computing the checksum in the process. - void storeHeader(UnpackedHeader *NewUnpackedHeader) { - NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader); + static INLINE void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) { + NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader); - AtomicPackedHeader *AtomicHeader = - reinterpret_cast<AtomicPackedHeader *>(this); - atomic_store_relaxed(AtomicHeader, NewPackedHeader); + atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); } // Packs and stores the header, computing the checksum in the process. We // compare the current header with the expected provided one to ensure that // we are not being raced by a corruption occurring in another thread. - void compareExchangeHeader(UnpackedHeader *NewUnpackedHeader, - UnpackedHeader *OldUnpackedHeader) { - NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader); + static INLINE void compareExchangeHeader(void *Ptr, + UnpackedHeader *NewUnpackedHeader, + UnpackedHeader *OldUnpackedHeader) { + NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader); PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader); - AtomicPackedHeader *AtomicHeader = - reinterpret_cast<AtomicPackedHeader *>(this); - if (UNLIKELY(!atomic_compare_exchange_strong(AtomicHeader, - &OldPackedHeader, - NewPackedHeader, - memory_order_relaxed))) { - dieWithMessage("ERROR: race on chunk header at address %p\n", this); + if (UNLIKELY(!atomic_compare_exchange_strong( + getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, + memory_order_relaxed))) { + dieWithMessage("ERROR: race on chunk header at address %p\n", Ptr); } } -}; - -ScudoChunk *getScudoChunk(uptr UserBeg) { - return reinterpret_cast<ScudoChunk *>(UserBeg - AlignedChunkHeaderSize); -} - -struct AllocatorOptions { - u32 QuarantineSizeMb; - u32 ThreadLocalQuarantineSizeKb; - bool MayReturnNull; - s32 ReleaseToOSIntervalMs; - bool DeallocationTypeMismatch; - bool DeleteSizeMismatch; - bool ZeroContents; - - void setFrom(const Flags *f, const CommonFlags *cf); - void copyTo(Flags *f, CommonFlags *cf) const; -}; - -void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) { - MayReturnNull = cf->allocator_may_return_null; - ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms; - QuarantineSizeMb = f->QuarantineSizeMb; - ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb; - DeallocationTypeMismatch = f->DeallocationTypeMismatch; - DeleteSizeMismatch = f->DeleteSizeMismatch; - ZeroContents = f->ZeroContents; -} - -void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const { - cf->allocator_may_return_null = MayReturnNull; - cf->allocator_release_to_os_interval_ms = ReleaseToOSIntervalMs; - f->QuarantineSizeMb = QuarantineSizeMb; - f->ThreadLocalQuarantineSizeKb = ThreadLocalQuarantineSizeKb; - f->DeallocationTypeMismatch = DeallocationTypeMismatch; - f->DeleteSizeMismatch = DeleteSizeMismatch; - f->ZeroContents = ZeroContents; -} - -static void initScudoInternal(const AllocatorOptions &Options); - -static bool ScudoInitIsRunning = false; - -void initScudo() { - SanitizerToolName = "Scudo"; - CHECK(!ScudoInitIsRunning && "Scudo init calls itself!"); - ScudoInitIsRunning = true; - - // Check if hardware CRC32 is supported in the binary and by the platform, if - // so, opt for the CRC32 hardware version of the checksum. - if (computeHardwareCRC32 && testCPUFeature(CRC32CPUFeature)) - atomic_store_relaxed(&HashAlgorithm, CRC32Hardware); - - initFlags(); - - AllocatorOptions Options; - Options.setFrom(getFlags(), common_flags()); - initScudoInternal(Options); - - // TODO(kostyak): determine if MaybeStartBackgroudThread could be of some use. - - ScudoInitIsRunning = false; -} +} // namespace Chunk struct QuarantineCallback { explicit QuarantineCallback(AllocatorCache *Cache) @@ -224,52 +171,46 @@ struct QuarantineCallback { // Chunk recycling function, returns a quarantined chunk to the backend, // first making sure it hasn't been tampered with. - void Recycle(ScudoChunk *Chunk) { + void Recycle(void *Ptr) { UnpackedHeader Header; - Chunk->loadHeader(&Header); + Chunk::loadHeader(Ptr, &Header); if (UNLIKELY(Header.State != ChunkQuarantine)) { dieWithMessage("ERROR: invalid chunk state when recycling address %p\n", - Chunk); + Ptr); } - Chunk->eraseHeader(); - void *Ptr = Chunk->getAllocBeg(&Header); - if (Header.FromPrimary) - getBackendAllocator().deallocatePrimary(Cache_, Ptr); + Chunk::eraseHeader(Ptr); + void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header); + if (Header.ClassId) + getBackendAllocator().deallocatePrimary(Cache_, BackendPtr, + Header.ClassId); else - getBackendAllocator().deallocateSecondary(Ptr); + getBackendAllocator().deallocateSecondary(BackendPtr); } // Internal quarantine allocation and deallocation functions. We first check // that the batches are indeed serviced by the Primary. // TODO(kostyak): figure out the best way to protect the batches. - COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize); void *Allocate(uptr Size) { - return getBackendAllocator().allocatePrimary(Cache_, Size); + return getBackendAllocator().allocatePrimary(Cache_, BatchClassId); } void Deallocate(void *Ptr) { - getBackendAllocator().deallocatePrimary(Cache_, Ptr); + getBackendAllocator().deallocatePrimary(Cache_, Ptr, BatchClassId); } AllocatorCache *Cache_; + COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize); + const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch)); }; -typedef Quarantine<QuarantineCallback, ScudoChunk> ScudoQuarantine; +typedef Quarantine<QuarantineCallback, void> ScudoQuarantine; typedef ScudoQuarantine::Cache ScudoQuarantineCache; COMPILER_CHECK(sizeof(ScudoQuarantineCache) <= - sizeof(ScudoThreadContext::QuarantineCachePlaceHolder)); + sizeof(ScudoTSD::QuarantineCachePlaceHolder)); -AllocatorCache *getAllocatorCache(ScudoThreadContext *ThreadContext) { - return &ThreadContext->Cache; -} - -ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) { - return reinterpret_cast< - ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder); -} - -ScudoPrng *getPrng(ScudoThreadContext *ThreadContext) { - return &ThreadContext->Prng; +ScudoQuarantineCache *getQuarantineCache(ScudoTSD *TSD) { + return reinterpret_cast<ScudoQuarantineCache *>( + TSD->QuarantineCachePlaceHolder); } struct ScudoAllocator { @@ -281,26 +222,22 @@ struct ScudoAllocator { ScudoBackendAllocator BackendAllocator; ScudoQuarantine AllocatorQuarantine; - StaticSpinMutex GlobalPrngMutex; - ScudoPrng GlobalPrng; - - // The fallback caches are used when the thread local caches have been - // 'detroyed' on thread tear-down. They are protected by a Mutex as they can - // be accessed by different threads. - StaticSpinMutex FallbackMutex; - AllocatorCache FallbackAllocatorCache; - ScudoQuarantineCache FallbackQuarantineCache; - ScudoPrng FallbackPrng; + u32 QuarantineChunksUpToSize; bool DeallocationTypeMismatch; bool ZeroContents; bool DeleteSizeMismatch; + bool CheckRssLimit; + uptr HardRssLimitMb; + uptr SoftRssLimitMb; + atomic_uint8_t RssLimitExceeded; + atomic_uint64_t RssLastCheckedAtNS; + explicit ScudoAllocator(LinkerInitialized) - : AllocatorQuarantine(LINKER_INITIALIZED), - FallbackQuarantineCache(LINKER_INITIALIZED) {} + : AllocatorQuarantine(LINKER_INITIALIZED) {} - void init(const AllocatorOptions &Options) { + void performSanityChecks() { // Verify that the header offset field can hold the maximum offset. In the // case of the Secondary allocator, it takes care of alignment and the // offset will always be 0. In the case of the Primary, the worst case @@ -310,9 +247,9 @@ struct ScudoAllocator { // result, the maximum offset will be at most the maximum alignment for the // last size class minus the header size, in multiples of MinAlignment. UnpackedHeader Header = {}; - uptr MaxPrimaryAlignment = + const uptr MaxPrimaryAlignment = 1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment); - uptr MaxOffset = + const uptr MaxOffset = (MaxPrimaryAlignment - AlignedChunkHeaderSize) >> MinAlignmentLog; Header.Offset = MaxOffset; if (Header.Offset != MaxOffset) { @@ -324,36 +261,97 @@ struct ScudoAllocator { // case scenario happens in the Primary. It will depend on the second to // last and last class sizes, as well as the dynamic base for the Primary. // The following is an over-approximation that works for our needs. - uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1; + const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1; Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes; if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) { dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in " "the header\n"); } - DeallocationTypeMismatch = Options.DeallocationTypeMismatch; - DeleteSizeMismatch = Options.DeleteSizeMismatch; - ZeroContents = Options.ZeroContents; - SetAllocatorMayReturnNull(Options.MayReturnNull); - BackendAllocator.init(Options.ReleaseToOSIntervalMs); + const uptr LargestClassId = SizeClassMap::kLargestClassID; + Header.ClassId = LargestClassId; + if (Header.ClassId != LargestClassId) { + dieWithMessage("ERROR: the largest class ID doesn't fit in the header\n"); + } + } + + void init() { + SanitizerToolName = "Scudo"; + initFlags(); + + performSanityChecks(); + + // Check if hardware CRC32 is supported in the binary and by the platform, + // if so, opt for the CRC32 hardware version of the checksum. + if (&computeHardwareCRC32 && hasHardwareCRC32()) + atomic_store_relaxed(&HashAlgorithm, CRC32Hardware); + + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms); + HardRssLimitMb = common_flags()->hard_rss_limit_mb; + SoftRssLimitMb = common_flags()->soft_rss_limit_mb; AllocatorQuarantine.Init( - static_cast<uptr>(Options.QuarantineSizeMb) << 20, - static_cast<uptr>(Options.ThreadLocalQuarantineSizeKb) << 10); - GlobalPrng.init(); - Cookie = GlobalPrng.getU64(); - BackendAllocator.initCache(&FallbackAllocatorCache); - FallbackPrng.init(); + static_cast<uptr>(getFlags()->QuarantineSizeKb) << 10, + static_cast<uptr>(getFlags()->ThreadLocalQuarantineSizeKb) << 10); + QuarantineChunksUpToSize = getFlags()->QuarantineChunksUpToSize; + DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch; + DeleteSizeMismatch = getFlags()->DeleteSizeMismatch; + ZeroContents = getFlags()->ZeroContents; + + if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(&Cookie), sizeof(Cookie), + /*blocking=*/false))) { + Cookie = static_cast<u32>((NanoTime() >> 12) ^ + (reinterpret_cast<uptr>(this) >> 4)); + } + + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + if (CheckRssLimit) + atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime()); } // Helper function that checks for a valid Scudo chunk. nullptr isn't. - bool isValidPointer(const void *UserPtr) { + bool isValidPointer(const void *Ptr) { initThreadMaybe(); - if (UNLIKELY(!UserPtr)) + if (UNLIKELY(!Ptr)) return false; - uptr UserBeg = reinterpret_cast<uptr>(UserPtr); - if (!IsAligned(UserBeg, MinAlignment)) + if (!Chunk::isAligned(Ptr)) return false; - return getScudoChunk(UserBeg)->isValid(); + return Chunk::isValid(Ptr); + } + + // Opportunistic RSS limit check. This will update the RSS limit status, if + // it can, every 100ms, otherwise it will just return the current one. + bool isRssLimitExceeded() { + u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); + const u64 CurrentCheck = MonotonicNanoTime(); + if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL))) + return atomic_load_relaxed(&RssLimitExceeded); + if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, + CurrentCheck, memory_order_relaxed)) + return atomic_load_relaxed(&RssLimitExceeded); + // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the + // RSS from /proc/self/statm by default. We might want to + // call getrusage directly, even if it's less accurate. + const uptr CurrentRssMb = GetRSS() >> 20; + if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) { + Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, HardRssLimitMb, CurrentRssMb); + DumpProcessMap(); + Die(); + } + if (SoftRssLimitMb) { + if (atomic_load_relaxed(&RssLimitExceeded)) { + if (CurrentRssMb <= SoftRssLimitMb) + atomic_store_relaxed(&RssLimitExceeded, false); + } else { + if (CurrentRssMb > SoftRssLimitMb) { + atomic_store_relaxed(&RssLimitExceeded, true); + Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, SoftRssLimitMb, CurrentRssMb); + } + } + } + return atomic_load_relaxed(&RssLimitExceeded); } // Allocates a chunk. @@ -375,207 +373,185 @@ struct ScudoAllocator { if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) return FailureHandler::OnBadRequest(); + if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) + return FailureHandler::OnOOM(); + // Primary and Secondary backed allocations have a different treatment. We // deal with alignment requirements of Primary serviced allocations here, // but the Secondary will take care of its own alignment needs. - bool FromPrimary = PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment); - - void *Ptr; - u8 Salt; - uptr AllocSize; - if (FromPrimary) { - AllocSize = AlignedSize; - ScudoThreadContext *ThreadContext = getThreadContextAndLock(); - if (LIKELY(ThreadContext)) { - Salt = getPrng(ThreadContext)->getU8(); - Ptr = BackendAllocator.allocatePrimary(getAllocatorCache(ThreadContext), - AllocSize); - ThreadContext->unlock(); - } else { - SpinMutexLock l(&FallbackMutex); - Salt = FallbackPrng.getU8(); - Ptr = BackendAllocator.allocatePrimary(&FallbackAllocatorCache, - AllocSize); - } + void *BackendPtr; + uptr BackendSize; + u8 ClassId; + if (PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment)) { + BackendSize = AlignedSize; + ClassId = SizeClassMap::ClassID(BackendSize); + ScudoTSD *TSD = getTSDAndLock(); + BackendPtr = BackendAllocator.allocatePrimary(&TSD->Cache, ClassId); + TSD->unlock(); } else { - { - SpinMutexLock l(&GlobalPrngMutex); - Salt = GlobalPrng.getU8(); - } - AllocSize = NeededSize; - Ptr = BackendAllocator.allocateSecondary(AllocSize, Alignment); + BackendSize = NeededSize; + ClassId = 0; + BackendPtr = BackendAllocator.allocateSecondary(BackendSize, Alignment); } - if (UNLIKELY(!Ptr)) + if (UNLIKELY(!BackendPtr)) return FailureHandler::OnOOM(); // If requested, we will zero out the entire contents of the returned chunk. - if ((ForceZeroContents || ZeroContents) && FromPrimary) - memset(Ptr, 0, BackendAllocator.getActuallyAllocatedSize( - Ptr, /*FromPrimary=*/true)); + if ((ForceZeroContents || ZeroContents) && ClassId) + memset(BackendPtr, 0, + BackendAllocator.getActuallyAllocatedSize(BackendPtr, ClassId)); UnpackedHeader Header = {}; - uptr AllocBeg = reinterpret_cast<uptr>(Ptr); - uptr UserBeg = AllocBeg + AlignedChunkHeaderSize; - if (UNLIKELY(!IsAligned(UserBeg, Alignment))) { + uptr UserPtr = reinterpret_cast<uptr>(BackendPtr) + AlignedChunkHeaderSize; + if (UNLIKELY(!IsAligned(UserPtr, Alignment))) { // Since the Secondary takes care of alignment, a non-aligned pointer // means it is from the Primary. It is also the only case where the offset // field of the header would be non-zero. - CHECK(FromPrimary); - UserBeg = RoundUpTo(UserBeg, Alignment); - uptr Offset = UserBeg - AlignedChunkHeaderSize - AllocBeg; - Header.Offset = Offset >> MinAlignmentLog; + DCHECK(ClassId); + const uptr AlignedUserPtr = RoundUpTo(UserPtr, Alignment); + Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog; + UserPtr = AlignedUserPtr; } - CHECK_LE(UserBeg + Size, AllocBeg + AllocSize); + CHECK_LE(UserPtr + Size, reinterpret_cast<uptr>(BackendPtr) + BackendSize); Header.State = ChunkAllocated; Header.AllocType = Type; - if (FromPrimary) { - Header.FromPrimary = 1; + if (ClassId) { + Header.ClassId = ClassId; Header.SizeOrUnusedBytes = Size; } else { // The secondary fits the allocations to a page, so the amount of unused // bytes is the difference between the end of the user allocation and the // next page boundary. - uptr PageSize = GetPageSizeCached(); - uptr TrailingBytes = (UserBeg + Size) & (PageSize - 1); + const uptr PageSize = GetPageSizeCached(); + const uptr TrailingBytes = (UserPtr + Size) & (PageSize - 1); if (TrailingBytes) Header.SizeOrUnusedBytes = PageSize - TrailingBytes; } - Header.Salt = Salt; - getScudoChunk(UserBeg)->storeHeader(&Header); - void *UserPtr = reinterpret_cast<void *>(UserBeg); - // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(UserPtr, Size); - return UserPtr; + void *Ptr = reinterpret_cast<void *>(UserPtr); + Chunk::storeHeader(Ptr, &Header); + // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(Ptr, Size); + return Ptr; } - // Place a chunk in the quarantine. In the event of a zero-sized quarantine, - // we directly deallocate the chunk, otherwise the flow would lead to the - // chunk being loaded (and checked) twice, and stored (and checksummed) once, - // with no additional security value. - void quarantineOrDeallocateChunk(ScudoChunk *Chunk, UnpackedHeader *Header, + // Place a chunk in the quarantine or directly deallocate it in the event of + // a zero-sized quarantine, or if the size of the chunk is greater than the + // quarantine chunk size threshold. + void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header, uptr Size) { - bool FromPrimary = Header->FromPrimary; - bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0); + const bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0) || + (Size > QuarantineChunksUpToSize); if (BypassQuarantine) { - Chunk->eraseHeader(); - void *Ptr = Chunk->getAllocBeg(Header); - if (FromPrimary) { - ScudoThreadContext *ThreadContext = getThreadContextAndLock(); - if (LIKELY(ThreadContext)) { - getBackendAllocator().deallocatePrimary( - getAllocatorCache(ThreadContext), Ptr); - ThreadContext->unlock(); - } else { - SpinMutexLock Lock(&FallbackMutex); - getBackendAllocator().deallocatePrimary(&FallbackAllocatorCache, Ptr); - } + Chunk::eraseHeader(Ptr); + void *BackendPtr = Chunk::getBackendPtr(Ptr, Header); + if (Header->ClassId) { + ScudoTSD *TSD = getTSDAndLock(); + getBackendAllocator().deallocatePrimary(&TSD->Cache, BackendPtr, + Header->ClassId); + TSD->unlock(); } else { - getBackendAllocator().deallocateSecondary(Ptr); + getBackendAllocator().deallocateSecondary(BackendPtr); } } else { + // If a small memory amount was allocated with a larger alignment, we want + // to take that into account. Otherwise the Quarantine would be filled + // with tiny chunks, taking a lot of VA memory. This is an approximation + // of the usable size, that allows us to not call + // GetActuallyAllocatedSize. + uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog); UnpackedHeader NewHeader = *Header; NewHeader.State = ChunkQuarantine; - Chunk->compareExchangeHeader(&NewHeader, Header); - ScudoThreadContext *ThreadContext = getThreadContextAndLock(); - if (LIKELY(ThreadContext)) { - AllocatorQuarantine.Put(getQuarantineCache(ThreadContext), - QuarantineCallback( - getAllocatorCache(ThreadContext)), - Chunk, Size); - ThreadContext->unlock(); - } else { - SpinMutexLock l(&FallbackMutex); - AllocatorQuarantine.Put(&FallbackQuarantineCache, - QuarantineCallback(&FallbackAllocatorCache), - Chunk, Size); - } + Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); + ScudoTSD *TSD = getTSDAndLock(); + AllocatorQuarantine.Put(getQuarantineCache(TSD), + QuarantineCallback(&TSD->Cache), Ptr, + EstimatedSize); + TSD->unlock(); } } - // Deallocates a Chunk, which means adding it to the delayed free list (or - // Quarantine). - void deallocate(void *UserPtr, uptr DeleteSize, AllocType Type) { - initThreadMaybe(); - // if (&__sanitizer_free_hook) __sanitizer_free_hook(UserPtr); - if (UNLIKELY(!UserPtr)) + // Deallocates a Chunk, which means either adding it to the quarantine or + // directly returning it to the backend if criteria are met. + void deallocate(void *Ptr, uptr DeleteSize, AllocType Type) { + // For a deallocation, we only ensure minimal initialization, meaning thread + // local data will be left uninitialized for now (when using ELF TLS). The + // fallback cache will be used instead. This is a workaround for a situation + // where the only heap operation performed in a thread would be a free past + // the TLS destructors, ending up in initialized thread specific data never + // being destroyed properly. Any other heap operation will do a full init. + initThreadMaybe(/*MinimalInit=*/true); + // if (&__sanitizer_free_hook) __sanitizer_free_hook(Ptr); + if (UNLIKELY(!Ptr)) return; - uptr UserBeg = reinterpret_cast<uptr>(UserPtr); - if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) { + if (UNLIKELY(!Chunk::isAligned(Ptr))) { dieWithMessage("ERROR: attempted to deallocate a chunk not properly " - "aligned at address %p\n", UserPtr); + "aligned at address %p\n", Ptr); } - ScudoChunk *Chunk = getScudoChunk(UserBeg); - UnpackedHeader OldHeader; - Chunk->loadHeader(&OldHeader); - if (UNLIKELY(OldHeader.State != ChunkAllocated)) { + UnpackedHeader Header; + Chunk::loadHeader(Ptr, &Header); + if (UNLIKELY(Header.State != ChunkAllocated)) { dieWithMessage("ERROR: invalid chunk state when deallocating address " - "%p\n", UserPtr); + "%p\n", Ptr); } if (DeallocationTypeMismatch) { // The deallocation type has to match the allocation one. - if (OldHeader.AllocType != Type) { + if (Header.AllocType != Type) { // With the exception of memalign'd Chunks, that can be still be free'd. - if (OldHeader.AllocType != FromMemalign || Type != FromMalloc) { - dieWithMessage("ERROR: allocation type mismatch on address %p\n", - UserPtr); + if (Header.AllocType != FromMemalign || Type != FromMalloc) { + dieWithMessage("ERROR: allocation type mismatch when deallocating " + "address %p\n", Ptr); } } } - uptr Size = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes : - Chunk->getUsableSize(&OldHeader) - OldHeader.SizeOrUnusedBytes; + uptr Size = Header.ClassId ? Header.SizeOrUnusedBytes : + Chunk::getUsableSize(Ptr, &Header) - Header.SizeOrUnusedBytes; if (DeleteSizeMismatch) { if (DeleteSize && DeleteSize != Size) { dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n", - UserPtr); + Ptr); } } - - // If a small memory amount was allocated with a larger alignment, we want - // to take that into account. Otherwise the Quarantine would be filled with - // tiny chunks, taking a lot of VA memory. This is an approximation of the - // usable size, that allows us to not call GetActuallyAllocatedSize. - uptr LiableSize = Size + (OldHeader.Offset << MinAlignment); - quarantineOrDeallocateChunk(Chunk, &OldHeader, LiableSize); + quarantineOrDeallocateChunk(Ptr, &Header, Size); } // Reallocates a chunk. We can save on a new allocation if the new requested // size still fits in the chunk. void *reallocate(void *OldPtr, uptr NewSize) { initThreadMaybe(); - uptr UserBeg = reinterpret_cast<uptr>(OldPtr); - if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) { + if (UNLIKELY(!Chunk::isAligned(OldPtr))) { dieWithMessage("ERROR: attempted to reallocate a chunk not properly " "aligned at address %p\n", OldPtr); } - ScudoChunk *Chunk = getScudoChunk(UserBeg); UnpackedHeader OldHeader; - Chunk->loadHeader(&OldHeader); + Chunk::loadHeader(OldPtr, &OldHeader); if (UNLIKELY(OldHeader.State != ChunkAllocated)) { dieWithMessage("ERROR: invalid chunk state when reallocating address " "%p\n", OldPtr); } - if (UNLIKELY(OldHeader.AllocType != FromMalloc)) { - dieWithMessage("ERROR: invalid chunk type when reallocating address %p\n", - OldPtr); + if (DeallocationTypeMismatch) { + if (UNLIKELY(OldHeader.AllocType != FromMalloc)) { + dieWithMessage("ERROR: allocation type mismatch when reallocating " + "address %p\n", OldPtr); + } } - uptr UsableSize = Chunk->getUsableSize(&OldHeader); + const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader); // The new size still fits in the current chunk, and the size difference // is reasonable. if (NewSize <= UsableSize && (UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) { UnpackedHeader NewHeader = OldHeader; NewHeader.SizeOrUnusedBytes = - OldHeader.FromPrimary ? NewSize : UsableSize - NewSize; - Chunk->compareExchangeHeader(&NewHeader, &OldHeader); + OldHeader.ClassId ? NewSize : UsableSize - NewSize; + Chunk::compareExchangeHeader(OldPtr, &NewHeader, &OldHeader); return OldPtr; } // Otherwise, we have to allocate a new chunk and copy the contents of the // old one. void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); if (NewPtr) { - uptr OldSize = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes : + uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes : UsableSize - OldHeader.SizeOrUnusedBytes; - memcpy(NewPtr, OldPtr, Min(NewSize, OldSize)); - quarantineOrDeallocateChunk(Chunk, &OldHeader, UsableSize); + memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize)); + quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize); } return NewPtr; } @@ -585,16 +561,14 @@ struct ScudoAllocator { initThreadMaybe(); if (UNLIKELY(!Ptr)) return 0; - uptr UserBeg = reinterpret_cast<uptr>(Ptr); - ScudoChunk *Chunk = getScudoChunk(UserBeg); UnpackedHeader Header; - Chunk->loadHeader(&Header); + Chunk::loadHeader(Ptr, &Header); // Getting the usable size of a chunk only makes sense if it's allocated. if (UNLIKELY(Header.State != ChunkAllocated)) { dieWithMessage("ERROR: invalid chunk state when sizing address %p\n", Ptr); } - return Chunk->getUsableSize(&Header); + return Chunk::getUsableSize(Ptr, &Header); } void *calloc(uptr NMemB, uptr Size) { @@ -604,11 +578,10 @@ struct ScudoAllocator { return allocate(NMemB * Size, MinAlignment, FromMalloc, true); } - void commitBack(ScudoThreadContext *ThreadContext) { - AllocatorCache *Cache = getAllocatorCache(ThreadContext); - AllocatorQuarantine.Drain(getQuarantineCache(ThreadContext), - QuarantineCallback(Cache)); - BackendAllocator.destroyCache(Cache); + void commitBack(ScudoTSD *TSD) { + AllocatorQuarantine.Drain(getQuarantineCache(TSD), + QuarantineCallback(&TSD->Cache)); + BackendAllocator.destroyCache(&TSD->Cache); } uptr getStats(AllocatorStat StatType) { @@ -617,6 +590,19 @@ struct ScudoAllocator { BackendAllocator.getStats(stats); return stats[StatType]; } + + void *handleBadRequest() { + initThreadMaybe(); + return FailureHandler::OnBadRequest(); + } + + void setRssLimit(uptr LimitMb, bool HardLimit) { + if (HardLimit) + HardRssLimitMb = LimitMb; + else + SoftRssLimitMb = LimitMb; + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + } }; static ScudoAllocator Instance(LINKER_INITIALIZED); @@ -625,17 +611,17 @@ static ScudoBackendAllocator &getBackendAllocator() { return Instance.BackendAllocator; } -static void initScudoInternal(const AllocatorOptions &Options) { - Instance.init(Options); +void initScudo() { + Instance.init(); } -void ScudoThreadContext::init() { +void ScudoTSD::init(bool Shared) { + UnlockRequired = Shared; getBackendAllocator().initCache(&Cache); - Prng.init(); memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder)); } -void ScudoThreadContext::commitBack() { +void ScudoTSD::commitBack() { Instance.commitBack(this); } @@ -672,6 +658,10 @@ void *scudoValloc(uptr Size) { void *scudoPvalloc(uptr Size) { uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) { + errno = ENOMEM; + return Instance.handleBadRequest(); + } // pvalloc(0) should allocate one page. Size = Size ? RoundUpTo(Size, PageSize) : PageSize; return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign)); @@ -679,28 +669,28 @@ void *scudoPvalloc(uptr Size) { void *scudoMemalign(uptr Alignment, uptr Size) { if (UNLIKELY(!IsPowerOfTwo(Alignment))) { - errno = errno_EINVAL; - return ScudoAllocator::FailureHandler::OnBadRequest(); + errno = EINVAL; + return Instance.handleBadRequest(); } return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMemalign)); } int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) { if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) { - ScudoAllocator::FailureHandler::OnBadRequest(); - return errno_EINVAL; + Instance.handleBadRequest(); + return EINVAL; } void *Ptr = Instance.allocate(Size, Alignment, FromMemalign); if (UNLIKELY(!Ptr)) - return errno_ENOMEM; + return ENOMEM; *MemPtr = Ptr; return 0; } void *scudoAlignedAlloc(uptr Alignment, uptr Size) { if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) { - errno = errno_EINVAL; - return ScudoAllocator::FailureHandler::OnBadRequest(); + errno = EINVAL; + return Instance.handleBadRequest(); } return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc)); } @@ -742,3 +732,13 @@ int __sanitizer_get_ownership(const void *Ptr) { uptr __sanitizer_get_allocated_size(const void *Ptr) { return Instance.getUsableSize(Ptr); } + +// Interface functions + +extern "C" { +void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit) { // NOLINT + if (!SCUDO_CAN_USE_PUBLIC_INTERFACE) + return; + Instance.setRssLimit(LimitMb, !!HardLimit); +} +} // extern "C" diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.h b/contrib/compiler-rt/lib/scudo/scudo_allocator.h index 29d85995a3ee..a561247def9c 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.h @@ -14,21 +14,15 @@ #ifndef SCUDO_ALLOCATOR_H_ #define SCUDO_ALLOCATOR_H_ -#include "scudo_flags.h" - -#include "sanitizer_common/sanitizer_allocator.h" - -#if !SANITIZER_LINUX -# error "The Scudo hardened allocator is currently only supported on Linux." -#endif +#include "scudo_platform.h" namespace __scudo { enum AllocType : u8 { - FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc. - FromNew = 1, // Memory block came from operator new. - FromNewArray = 2, // Memory block came from operator new []. - FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc. + FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc. + FromNew = 1, // Memory block came from operator new. + FromNewArray = 2, // Memory block came from operator new []. + FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc. }; enum ChunkState : u8 { @@ -45,16 +39,15 @@ enum ChunkState : u8 { typedef u64 PackedHeader; struct UnpackedHeader { u64 Checksum : 16; - u64 SizeOrUnusedBytes : 19; // Size for Primary backed allocations, amount of - // unused bytes in the chunk for Secondary ones. - u64 FromPrimary : 1; - u64 State : 2; // available, allocated, or quarantined - u64 AllocType : 2; // malloc, new, new[], or memalign - u64 Offset : 16; // Offset from the beginning of the backend - // allocation to the beginning of the chunk - // itself, in multiples of MinAlignment. See - // comment about its maximum value and in init(). - u64 Salt : 8; + u64 ClassId : 8; + u64 SizeOrUnusedBytes : 20; // Size for Primary backed allocations, amount of + // unused bytes in the chunk for Secondary ones. + u64 State : 2; // available, allocated, or quarantined + u64 AllocType : 2; // malloc, new, new[], or memalign + u64 Offset : 16; // Offset from the beginning of the backend + // allocation to the beginning of the chunk + // itself, in multiples of MinAlignment. See + // comment about its maximum value and in init(). }; typedef atomic_uint64_t AtomicPackedHeader; @@ -72,14 +65,6 @@ const uptr AlignedChunkHeaderSize = #if SANITIZER_CAN_USE_ALLOCATOR64 const uptr AllocatorSpace = ~0ULL; -# if defined(__aarch64__) && SANITIZER_ANDROID -const uptr AllocatorSize = 0x4000000000ULL; // 256G. -# elif defined(__aarch64__) -const uptr AllocatorSize = 0x10000000000ULL; // 1T. -# else -const uptr AllocatorSize = 0x40000000000ULL; // 4T. -# endif -typedef DefaultSizeClassMap SizeClassMap; struct AP64 { static const uptr kSpaceBeg = AllocatorSpace; static const uptr kSpaceSize = AllocatorSize; @@ -91,17 +76,12 @@ struct AP64 { }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; #else -// Currently, the 32-bit Sanitizer allocator has not yet benefited from all the -// security improvements brought to the 64-bit one. This makes the 32-bit -// version of Scudo slightly less toughened. -static const uptr RegionSizeLog = 20; static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog; # if SANITIZER_WORDSIZE == 32 typedef FlatByteMap<NumRegions> ByteMap; # elif SANITIZER_WORDSIZE == 64 typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap; # endif // SANITIZER_WORDSIZE -typedef DefaultSizeClassMap SizeClassMap; struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; @@ -111,7 +91,8 @@ struct AP32 { typedef __scudo::ByteMap ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = - SizeClassAllocator32FlagMasks::kRandomShuffleChunks; + SizeClassAllocator32FlagMasks::kRandomShuffleChunks | + SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch; }; typedef SizeClassAllocator32<AP32> PrimaryAllocator; #endif // SANITIZER_CAN_USE_ALLOCATOR64 diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h b/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h index 7599c12abb6d..25e273114c23 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h @@ -31,8 +31,8 @@ class ScudoCombinedAllocator { // Primary allocations are always MinAlignment aligned, and as such do not // require an Alignment parameter. - void *allocatePrimary(AllocatorCache *Cache, uptr Size) { - return Cache->Allocate(&Primary, Primary.ClassID(Size)); + void *allocatePrimary(AllocatorCache *Cache, uptr ClassId) { + return Cache->Allocate(&Primary, ClassId); } // Secondary allocations do not require a Cache, but do require an Alignment @@ -41,17 +41,17 @@ class ScudoCombinedAllocator { return Secondary.Allocate(&Stats, Size, Alignment); } - void deallocatePrimary(AllocatorCache *Cache, void *Ptr) { - Cache->Deallocate(&Primary, Primary.GetSizeClass(Ptr), Ptr); + void deallocatePrimary(AllocatorCache *Cache, void *Ptr, uptr ClassId) { + Cache->Deallocate(&Primary, ClassId, Ptr); } void deallocateSecondary(void *Ptr) { Secondary.Deallocate(&Stats, Ptr); } - uptr getActuallyAllocatedSize(void *Ptr, bool FromPrimary) { - if (FromPrimary) - return PrimaryAllocator::ClassIdToSize(Primary.GetSizeClass(Ptr)); + uptr getActuallyAllocatedSize(void *Ptr, uptr ClassId) { + if (ClassId) + return PrimaryAllocator::ClassIdToSize(ClassId); return Secondary.GetActuallyAllocatedSize(Ptr); } diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h b/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h index dbfb22565f9c..f2002ed986c3 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h @@ -23,23 +23,24 @@ class ScudoLargeMmapAllocator { public: - void Init() { - PageSize = GetPageSizeCached(); + PageSizeCached = GetPageSizeCached(); } void *Allocate(AllocatorStats *Stats, uptr Size, uptr Alignment) { - uptr UserSize = Size - AlignedChunkHeaderSize; + const uptr UserSize = Size - AlignedChunkHeaderSize; // The Scudo frontend prevents us from allocating more than // MaxAllowedMallocSize, so integer overflow checks would be superfluous. - uptr MapSize = Size + SecondaryHeaderSize; + uptr MapSize = Size + AlignedReservedAddressRangeSize; if (Alignment > MinAlignment) MapSize += Alignment; + const uptr PageSize = PageSizeCached; MapSize = RoundUpTo(MapSize, PageSize); // Account for 2 guard pages, one before and one after the chunk. MapSize += 2 * PageSize; - uptr MapBeg = reinterpret_cast<uptr>(MmapNoAccess(MapSize)); + ReservedAddressRange AddressRange; + uptr MapBeg = AddressRange.Init(MapSize); if (MapBeg == ~static_cast<uptr>(0)) return ReturnNullOrDieOnFailure::OnOOM(); // A page-aligned pointer is assumed after that, so check it now. @@ -62,27 +63,27 @@ class ScudoLargeMmapAllocator { PageSize; CHECK_GE(NewMapBeg, MapBeg); if (NewMapBeg != MapBeg) { - UnmapOrDie(reinterpret_cast<void *>(MapBeg), NewMapBeg - MapBeg); + AddressRange.Unmap(MapBeg, NewMapBeg - MapBeg); MapBeg = NewMapBeg; } UserEnd = UserBeg + UserSize; } uptr NewMapEnd = RoundUpTo(UserEnd, PageSize) + PageSize; if (NewMapEnd != MapEnd) { - UnmapOrDie(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd); + AddressRange.Unmap(NewMapEnd, MapEnd - NewMapEnd); MapEnd = NewMapEnd; } MapSize = MapEnd - MapBeg; } CHECK_LE(UserEnd, MapEnd - PageSize); - // Actually mmap the memory, preserving the guard pages on either side. - CHECK_EQ(MapBeg + PageSize, reinterpret_cast<uptr>( - MmapFixedOrDie(MapBeg + PageSize, MapSize - 2 * PageSize))); - uptr Ptr = UserBeg - AlignedChunkHeaderSize; - SecondaryHeader *Header = getHeader(Ptr); - Header->MapBeg = MapBeg; - Header->MapSize = MapSize; + // Actually mmap the memory, preserving the guard pages on either side + CHECK_EQ(MapBeg + PageSize, + AddressRange.Map(MapBeg + PageSize, MapSize - 2 * PageSize)); + const uptr Ptr = UserBeg - AlignedChunkHeaderSize; + ReservedAddressRange *StoredRange = getReservedAddressRange(Ptr); + *StoredRange = AddressRange; + // The primary adds the whole class size to the stats when allocating a // chunk, so we will do something similar here. But we will not account for // the guard pages. @@ -96,42 +97,43 @@ class ScudoLargeMmapAllocator { } void Deallocate(AllocatorStats *Stats, void *Ptr) { - SecondaryHeader *Header = getHeader(Ptr); + // Since we're unmapping the entirety of where the ReservedAddressRange + // actually is, copy onto the stack. + const uptr PageSize = PageSizeCached; + ReservedAddressRange AddressRange = *getReservedAddressRange(Ptr); { SpinMutexLock l(&StatsMutex); - Stats->Sub(AllocatorStatAllocated, Header->MapSize - 2 * PageSize); - Stats->Sub(AllocatorStatMapped, Header->MapSize - 2 * PageSize); + Stats->Sub(AllocatorStatAllocated, AddressRange.size() - 2 * PageSize); + Stats->Sub(AllocatorStatMapped, AddressRange.size() - 2 * PageSize); } - UnmapOrDie(reinterpret_cast<void *>(Header->MapBeg), Header->MapSize); + AddressRange.Unmap(reinterpret_cast<uptr>(AddressRange.base()), + AddressRange.size()); } uptr GetActuallyAllocatedSize(void *Ptr) { - SecondaryHeader *Header = getHeader(Ptr); - // Deduct PageSize as MapSize includes the trailing guard page. - uptr MapEnd = Header->MapBeg + Header->MapSize - PageSize; + ReservedAddressRange *StoredRange = getReservedAddressRange(Ptr); + // Deduct PageSize as ReservedAddressRange size includes the trailing guard + // page. + uptr MapEnd = reinterpret_cast<uptr>(StoredRange->base()) + + StoredRange->size() - PageSizeCached; return MapEnd - reinterpret_cast<uptr>(Ptr); } private: - // A Secondary allocated chunk header contains the base of the mapping and - // its size, which comprises the guard pages. - struct SecondaryHeader { - uptr MapBeg; - uptr MapSize; - }; - // Check that sizeof(SecondaryHeader) is a multiple of MinAlignment. - COMPILER_CHECK((sizeof(SecondaryHeader) & (MinAlignment - 1)) == 0); - - SecondaryHeader *getHeader(uptr Ptr) { - return reinterpret_cast<SecondaryHeader*>(Ptr - sizeof(SecondaryHeader)); + ReservedAddressRange *getReservedAddressRange(uptr Ptr) { + return reinterpret_cast<ReservedAddressRange*>( + Ptr - sizeof(ReservedAddressRange)); } - SecondaryHeader *getHeader(const void *Ptr) { - return getHeader(reinterpret_cast<uptr>(Ptr)); + ReservedAddressRange *getReservedAddressRange(const void *Ptr) { + return getReservedAddressRange(reinterpret_cast<uptr>(Ptr)); } - const uptr SecondaryHeaderSize = sizeof(SecondaryHeader); - const uptr HeadersSize = SecondaryHeaderSize + AlignedChunkHeaderSize; - uptr PageSize; + static constexpr uptr AlignedReservedAddressRangeSize = + (sizeof(ReservedAddressRange) + MinAlignment - 1) & ~(MinAlignment - 1); + static constexpr uptr HeadersSize = + AlignedReservedAddressRangeSize + AlignedChunkHeaderSize; + + uptr PageSizeCached; SpinMutex StatsMutex; }; diff --git a/contrib/compiler-rt/lib/scudo/scudo_crc32.h b/contrib/compiler-rt/lib/scudo/scudo_crc32.h index 5ffcc62658cc..e89e430f4085 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_crc32.h +++ b/contrib/compiler-rt/lib/scudo/scudo_crc32.h @@ -40,7 +40,7 @@ enum : u8 { CRC32Hardware = 1, }; -const static u32 CRC32Table[] = { +static const u32 CRC32Table[] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, diff --git a/contrib/compiler-rt/lib/scudo/scudo_flags.cpp b/contrib/compiler-rt/lib/scudo/scudo_flags.cpp index 90f0cbf4bb86..2aff3ef1e8fa 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_flags.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_flags.cpp @@ -17,12 +17,11 @@ #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" -extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -const char* __scudo_default_options(); +SANITIZER_INTERFACE_WEAK_DEF(const char*, __scudo_default_options, void); namespace __scudo { -Flags ScudoFlags; // Use via getFlags(). +static Flags ScudoFlags; // Use via getFlags(). void Flags::setDefaults() { #define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; @@ -37,7 +36,7 @@ static void RegisterScudoFlags(FlagParser *parser, Flags *f) { #undef SCUDO_FLAG } -static const char *callGetScudoDefaultOptions() { +static const char *getScudoDefaultOptions() { return (&__scudo_default_options) ? __scudo_default_options() : ""; } @@ -57,8 +56,7 @@ void initFlags() { RegisterCommonFlags(&ScudoParser); // Override from user-specified string. - const char *ScudoDefaultOptions = callGetScudoDefaultOptions(); - ScudoParser.ParseString(ScudoDefaultOptions); + ScudoParser.ParseString(getScudoDefaultOptions()); // Override from environment. ScudoParser.ParseString(GetEnv("SCUDO_OPTIONS")); @@ -67,27 +65,52 @@ void initFlags() { // Sanity checks and default settings for the Quarantine parameters. - if (f->QuarantineSizeMb < 0) { - const int DefaultQuarantineSizeMb = FIRST_32_SECOND_64(4, 16); - f->QuarantineSizeMb = DefaultQuarantineSizeMb; + if (f->QuarantineSizeMb >= 0) { + // Backward compatible logic if QuarantineSizeMb is set. + if (f->QuarantineSizeKb >= 0) { + dieWithMessage("ERROR: please use either QuarantineSizeMb (deprecated) " + "or QuarantineSizeKb, but not both\n"); + } + if (f->QuarantineChunksUpToSize >= 0) { + dieWithMessage("ERROR: QuarantineChunksUpToSize cannot be used in " + " conjunction with the deprecated QuarantineSizeMb option\n"); + } + // If everything is in order, update QuarantineSizeKb accordingly. + f->QuarantineSizeKb = f->QuarantineSizeMb * 1024; + } else { + // Otherwise proceed with the new options. + if (f->QuarantineSizeKb < 0) { + const int DefaultQuarantineSizeKb = FIRST_32_SECOND_64(64, 256); + f->QuarantineSizeKb = DefaultQuarantineSizeKb; + } + if (f->QuarantineChunksUpToSize < 0) { + const int DefaultQuarantineChunksUpToSize = FIRST_32_SECOND_64(512, 2048); + f->QuarantineChunksUpToSize = DefaultQuarantineChunksUpToSize; + } } - // We enforce an upper limit for the quarantine size of 4Gb. - if (f->QuarantineSizeMb > (4 * 1024)) { + + // We enforce an upper limit for the chunk quarantine threshold of 4Mb. + if (f->QuarantineChunksUpToSize > (4 * 1024 * 1024)) { + dieWithMessage("ERROR: the chunk quarantine threshold is too large\n"); + } + + // We enforce an upper limit for the quarantine size of 32Mb. + if (f->QuarantineSizeKb > (32 * 1024)) { dieWithMessage("ERROR: the quarantine size is too large\n"); } + if (f->ThreadLocalQuarantineSizeKb < 0) { - const int DefaultThreadLocalQuarantineSizeKb = - FIRST_32_SECOND_64(64, 256); + const int DefaultThreadLocalQuarantineSizeKb = FIRST_32_SECOND_64(16, 64); f->ThreadLocalQuarantineSizeKb = DefaultThreadLocalQuarantineSizeKb; } - // And an upper limit of 128Mb for the thread quarantine cache. - if (f->ThreadLocalQuarantineSizeKb > (128 * 1024)) { + // And an upper limit of 8Mb for the thread quarantine cache. + if (f->ThreadLocalQuarantineSizeKb > (8 * 1024)) { dieWithMessage("ERROR: the per thread quarantine cache size is too " - "large\n"); + "large\n"); } - if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeMb > 0) { + if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeKb > 0) { dieWithMessage("ERROR: ThreadLocalQuarantineSizeKb can be set to 0 only " - "when QuarantineSizeMb is set to 0\n"); + "when QuarantineSizeKb is set to 0\n"); } } diff --git a/contrib/compiler-rt/lib/scudo/scudo_flags.inc b/contrib/compiler-rt/lib/scudo/scudo_flags.inc index 45f9ea846e1a..f180478fdac3 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_flags.inc +++ b/contrib/compiler-rt/lib/scudo/scudo_flags.inc @@ -15,17 +15,27 @@ # error "Define SCUDO_FLAG prior to including this file!" #endif -// Default value is set in scudo_flags.cpp based on architecture. SCUDO_FLAG(int, QuarantineSizeMb, -1, - "Size (in Mb) of quarantine used to delay the actual deallocation " - "of chunks. Lower value may reduce memory usage but decrease the " - "effectiveness of the mitigation.") + "Deprecated. Please use QuarantineSizeKb.") + +// Default value is set in scudo_flags.cpp based on architecture. +SCUDO_FLAG(int, QuarantineSizeKb, -1, + "Size in KB of quarantine used to delay the actual deallocation of " + "chunks. Lower value may reduce memory usage but decrease the " + "effectiveness of the mitigation. Defaults to 64KB (32-bit) or " + "256KB (64-bit)") // Default value is set in scudo_flags.cpp based on architecture. SCUDO_FLAG(int, ThreadLocalQuarantineSizeKb, -1, - "Size (in Kb) of per-thread cache used to offload the global " + "Size in KB of per-thread cache used to offload the global " "quarantine. Lower value may reduce memory usage but might increase " - "the contention on the global quarantine.") + "the contention on the global quarantine. Defaults to 16KB (32-bit) " + "or 64KB (64-bit)") + +// Default value is set in scudo_flags.cpp based on architecture. +SCUDO_FLAG(int, QuarantineChunksUpToSize, -1, + "Size in bytes up to which chunks will be quarantined (if lower than" + "or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)") SCUDO_FLAG(bool, DeallocationTypeMismatch, true, "Report errors on malloc/delete, new/free, new/delete[], etc.") diff --git a/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h b/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h new file mode 100644 index 000000000000..3f39e0c4ee0b --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h @@ -0,0 +1,22 @@ +//===-- scudo_interface_internal.h ------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Private Scudo interface header. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_INTERFACE_INTERNAL_H_ +#define SCUDO_INTERFACE_INTERNAL_H_ + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit); // NOLINT +} // extern "C" + +#endif // SCUDO_INTERFACE_INTERNAL_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp b/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp index cdefb127b965..c5a1abbed82b 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp @@ -15,7 +15,7 @@ #include "interception/interception.h" -#include <cstddef> +#include <stddef.h> using namespace __scudo; diff --git a/contrib/compiler-rt/lib/scudo/scudo_platform.h b/contrib/compiler-rt/lib/scudo/scudo_platform.h new file mode 100644 index 000000000000..e1c9c32e9a62 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_platform.h @@ -0,0 +1,80 @@ +//===-- scudo_platform.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo platform specific definitions. +/// TODO(kostyak): add tests for the compile time defines. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PLATFORM_H_ +#define SCUDO_PLATFORM_H_ + +#include "sanitizer_common/sanitizer_allocator.h" + +#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA +# error "The Scudo hardened allocator is not supported on this platform." +#endif + +#define SCUDO_TSD_EXCLUSIVE_SUPPORTED (!SANITIZER_ANDROID && !SANITIZER_FUCHSIA) + +#ifndef SCUDO_TSD_EXCLUSIVE +// SCUDO_TSD_EXCLUSIVE wasn't defined, use a default TSD model for the platform. +# if SANITIZER_ANDROID || SANITIZER_FUCHSIA +// Android and Fuchsia use a pool of TSDs shared between threads. +# define SCUDO_TSD_EXCLUSIVE 0 +# elif SANITIZER_LINUX && !SANITIZER_ANDROID +// Non-Android Linux use an exclusive TSD per thread. +# define SCUDO_TSD_EXCLUSIVE 1 +# else +# error "No default TSD model defined for this platform." +# endif // SANITIZER_ANDROID || SANITIZER_FUCHSIA +#endif // SCUDO_TSD_EXCLUSIVE + +// If the exclusive TSD model is chosen, make sure the platform supports it. +#if SCUDO_TSD_EXCLUSIVE && !SCUDO_TSD_EXCLUSIVE_SUPPORTED +# error "The exclusive TSD model is not supported on this platform." +#endif + +// Maximum number of TSDs that can be created for the Shared model. +#ifndef SCUDO_SHARED_TSD_POOL_SIZE +# define SCUDO_SHARED_TSD_POOL_SIZE 32U +#endif // SCUDO_SHARED_TSD_POOL_SIZE + +// The following allows the public interface functions to be disabled. +#ifndef SCUDO_CAN_USE_PUBLIC_INTERFACE +# define SCUDO_CAN_USE_PUBLIC_INTERFACE 1 +#endif + +namespace __scudo { + +#if SANITIZER_CAN_USE_ALLOCATOR64 +# if defined(__aarch64__) && SANITIZER_ANDROID +const uptr AllocatorSize = 0x4000000000ULL; // 256G. +# elif defined(__aarch64__) +const uptr AllocatorSize = 0x10000000000ULL; // 1T. +# else +const uptr AllocatorSize = 0x40000000000ULL; // 4T. +# endif +#else +const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20; +#endif // SANITIZER_CAN_USE_ALLOCATOR64 + +#if !defined(SCUDO_SIZE_CLASS_MAP) +# define SCUDO_SIZE_CLASS_MAP Default +#endif + +#define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP) +#define SIZE_CLASS_MAP_TYPE_(T) SIZE_CLASS_MAP_TYPE__(T) +#define SIZE_CLASS_MAP_TYPE__(T) T##SizeClassMap + +typedef SIZE_CLASS_MAP_TYPE SizeClassMap; + +} // namespace __scudo + +#endif // SCUDO_PLATFORM_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls.h b/contrib/compiler-rt/lib/scudo/scudo_tls.h deleted file mode 100644 index 20c49204cf13..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls.h +++ /dev/null @@ -1,47 +0,0 @@ -//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure definition. -/// Implementation will differ based on the thread local storage primitives -/// offered by the underlying platform. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_H_ -#define SCUDO_TLS_H_ - -#include "scudo_allocator.h" -#include "scudo_utils.h" - -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_platform.h" - -namespace __scudo { - -// Platform specific base thread context definitions. -#include "scudo_tls_context_android.inc" -#include "scudo_tls_context_linux.inc" - -struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform { - AllocatorCache Cache; - ScudoPrng Prng; - uptr QuarantineCachePlaceHolder[4]; - void init(); - void commitBack(); -}; - -void initThread(); - -// Platform specific dastpath functions definitions. -#include "scudo_tls_android.inc" -#include "scudo_tls_linux.inc" - -} // namespace __scudo - -#endif // SCUDO_TLS_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp b/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp deleted file mode 100644 index ec74e37c8dbc..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp +++ /dev/null @@ -1,95 +0,0 @@ -//===-- scudo_tls_android.cpp -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure implementation for Android. -/// -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -#include "scudo_tls.h" - -#include <pthread.h> - -namespace __scudo { - -static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; -static pthread_key_t PThreadKey; - -static atomic_uint32_t ThreadContextCurrentIndex; -static ScudoThreadContext *ThreadContexts; -static uptr NumberOfContexts; - -// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used as they allocate memory. -static uptr getNumberOfCPUs() { - cpu_set_t CPUs; - CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); - return CPU_COUNT(&CPUs); -} - -static void initOnce() { - // Hack: TLS_SLOT_TSAN was introduced in N. To be able to use it on M for - // testing, we create an unused key. Since the key_data array follows the tls - // array, it basically gives us the extra entry we need. - // TODO(kostyak): remove and restrict to N and above. - CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0); - initScudo(); - NumberOfContexts = getNumberOfCPUs(); - ThreadContexts = reinterpret_cast<ScudoThreadContext *>( - MmapOrDie(sizeof(ScudoThreadContext) * NumberOfContexts, __func__)); - for (uptr i = 0; i < NumberOfContexts; i++) - ThreadContexts[i].init(); -} - -void initThread() { - pthread_once(&GlobalInitialized, initOnce); - // Initial context assignment is done in a plain round-robin fashion. - u32 Index = atomic_fetch_add(&ThreadContextCurrentIndex, 1, - memory_order_relaxed); - ScudoThreadContext *ThreadContext = - &ThreadContexts[Index % NumberOfContexts]; - *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext); -} - -ScudoThreadContext *getThreadContextAndLockSlow() { - ScudoThreadContext *ThreadContext; - // Go through all the contexts and find the first unlocked one. - for (u32 i = 0; i < NumberOfContexts; i++) { - ThreadContext = &ThreadContexts[i]; - if (ThreadContext->tryLock()) { - *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext); - return ThreadContext; - } - } - // No luck, find the one with the lowest precedence, and slow lock it. - u64 Precedence = UINT64_MAX; - for (u32 i = 0; i < NumberOfContexts; i++) { - u64 SlowLockPrecedence = ThreadContexts[i].getSlowLockPrecedence(); - if (SlowLockPrecedence && SlowLockPrecedence < Precedence) { - ThreadContext = &ThreadContexts[i]; - Precedence = SlowLockPrecedence; - } - } - if (LIKELY(Precedence != UINT64_MAX)) { - ThreadContext->lock(); - *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext); - return ThreadContext; - } - // Last resort (can this happen?), stick with the current one. - ThreadContext = - reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr()); - ThreadContext->lock(); - return ThreadContext; -} - -} // namespace __scudo - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc deleted file mode 100644 index 8ecad7a30a6c..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc +++ /dev/null @@ -1,44 +0,0 @@ -//===-- scudo_tls_android.inc -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure fastpath functions implementation for Android. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_ANDROID_H_ -#define SCUDO_TLS_ANDROID_H_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -ALWAYS_INLINE void initThreadMaybe() { - if (LIKELY(*get_android_tls_ptr())) - return; - initThread(); -} - -ScudoThreadContext *getThreadContextAndLockSlow(); - -ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() { - ScudoThreadContext *ThreadContext = - reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr()); - CHECK(ThreadContext); - // Try to lock the currently associated context. - if (ThreadContext->tryLock()) - return ThreadContext; - // If it failed, go the slow path. - return getThreadContextAndLockSlow(); -} - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID - -#endif // SCUDO_TLS_ANDROID_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc deleted file mode 100644 index f1951319d487..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc +++ /dev/null @@ -1,54 +0,0 @@ -//===-- scudo_tls_context_android.inc ---------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Android specific base thread context definition. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_CONTEXT_ANDROID_INC_ -#define SCUDO_TLS_CONTEXT_ANDROID_INC_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -struct ScudoThreadContextPlatform { - INLINE bool tryLock() { - if (Mutex.TryLock()) { - atomic_store_relaxed(&SlowLockPrecedence, 0); - return true; - } - if (atomic_load_relaxed(&SlowLockPrecedence) == 0) - atomic_store_relaxed(&SlowLockPrecedence, NanoTime()); - return false; - } - - INLINE void lock() { - Mutex.Lock(); - atomic_store_relaxed(&SlowLockPrecedence, 0); - } - - INLINE void unlock() { - Mutex.Unlock(); - } - - INLINE u64 getSlowLockPrecedence() { - return atomic_load_relaxed(&SlowLockPrecedence); - } - - private: - StaticSpinMutex Mutex; - atomic_uint64_t SlowLockPrecedence; -}; - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID - -#endif // SCUDO_TLS_CONTEXT_ANDROID_INC_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc deleted file mode 100644 index 8d292bdbc932..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc +++ /dev/null @@ -1,29 +0,0 @@ -//===-- scudo_tls_context_linux.inc -----------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Linux specific base thread context definition. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_CONTEXT_LINUX_INC_ -#define SCUDO_TLS_CONTEXT_LINUX_INC_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -struct ScudoThreadContextPlatform { - ALWAYS_INLINE void unlock() {} -}; - -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID - -#endif // SCUDO_TLS_CONTEXT_LINUX_INC_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc deleted file mode 100644 index 242ee3329ea8..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc +++ /dev/null @@ -1,48 +0,0 @@ -//===-- scudo_tls_linux.inc -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure fastpath functions implementation for platforms -/// supporting thread_local. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_LINUX_H_ -#define SCUDO_TLS_LINUX_H_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -enum ThreadState : u8 { - ThreadNotInitialized = 0, - ThreadInitialized, - ThreadTornDown, -}; -__attribute__((tls_model("initial-exec"))) -extern THREADLOCAL ThreadState ScudoThreadState; -__attribute__((tls_model("initial-exec"))) -extern THREADLOCAL ScudoThreadContext ThreadLocalContext; - -ALWAYS_INLINE void initThreadMaybe() { - if (LIKELY(ScudoThreadState != ThreadNotInitialized)) - return; - initThread(); -} - -ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() { - if (UNLIKELY(ScudoThreadState == ThreadTornDown)) - return nullptr; - return &ThreadLocalContext; -} - -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID - -#endif // SCUDO_TLS_LINUX_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd.h b/contrib/compiler-rt/lib/scudo/scudo_tsd.h new file mode 100644 index 000000000000..80464b5ea1e4 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd.h @@ -0,0 +1,72 @@ +//===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo thread specific data definition. +/// Implementation will differ based on the thread local storage primitives +/// offered by the underlying platform. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +#define SCUDO_TSD_H_ + +#include "scudo_allocator.h" +#include "scudo_utils.h" + +#include <pthread.h> + +namespace __scudo { + +struct ALIGNED(64) ScudoTSD { + AllocatorCache Cache; + uptr QuarantineCachePlaceHolder[4]; + + void init(bool Shared); + void commitBack(); + + INLINE bool tryLock() { + if (Mutex.TryLock()) { + atomic_store_relaxed(&Precedence, 0); + return true; + } + if (atomic_load_relaxed(&Precedence) == 0) + atomic_store_relaxed(&Precedence, MonotonicNanoTime()); + return false; + } + + INLINE void lock() { + Mutex.Lock(); + atomic_store_relaxed(&Precedence, 0); + } + + INLINE void unlock() { + if (!UnlockRequired) + return; + Mutex.Unlock(); + } + + INLINE u64 getPrecedence() { + return atomic_load_relaxed(&Precedence); + } + + private: + bool UnlockRequired; + StaticSpinMutex Mutex; + atomic_uint64_t Precedence; +}; + +void initThread(bool MinimalInit); + +// TSD model specific fastpath functions definitions. +#include "scudo_tsd_exclusive.inc" +#include "scudo_tsd_shared.inc" + +} // namespace __scudo + +#endif // SCUDO_TSD_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp index 1e38233f339c..1084dfac91e1 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp @@ -1,4 +1,4 @@ -//===-- scudo_tls_linux.cpp -------------------------------------*- C++ -*-===// +//===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,18 +7,13 @@ // //===----------------------------------------------------------------------===// /// -/// Scudo thread local structure implementation for platforms supporting -/// thread_local. +/// Scudo exclusive TSD implementation. /// //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" +#include "scudo_tsd.h" -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -#include "scudo_tls.h" - -#include <pthread.h> +#if SCUDO_TSD_EXCLUSIVE namespace __scudo { @@ -28,7 +23,11 @@ static pthread_key_t PThreadKey; __attribute__((tls_model("initial-exec"))) THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized; __attribute__((tls_model("initial-exec"))) -THREADLOCAL ScudoThreadContext ThreadLocalContext; +THREADLOCAL ScudoTSD TSD; + +// Fallback TSD for when the thread isn't initialized yet or is torn down. It +// can be shared between multiple threads and as such must be locked. +ScudoTSD FallbackTSD; static void teardownThread(void *Ptr) { uptr I = reinterpret_cast<uptr>(Ptr); @@ -43,7 +42,7 @@ static void teardownThread(void *Ptr) { reinterpret_cast<void *>(I - 1)) == 0)) return; } - ThreadLocalContext.commitBack(); + TSD.commitBack(); ScudoThreadState = ThreadTornDown; } @@ -51,16 +50,19 @@ static void teardownThread(void *Ptr) { static void initOnce() { CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0); initScudo(); + FallbackTSD.init(/*Shared=*/true); } -void initThread() { +void initThread(bool MinimalInit) { CHECK_EQ(pthread_once(&GlobalInitialized, initOnce), 0); + if (UNLIKELY(MinimalInit)) + return; CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>( GetPthreadDestructorIterations())), 0); - ThreadLocalContext.init(); + TSD.init(/*Shared=*/false); ScudoThreadState = ThreadInitialized; } } // namespace __scudo -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc new file mode 100644 index 000000000000..567b6a1edd12 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc @@ -0,0 +1,46 @@ +//===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo exclusive TSD fastpath functions implementation. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +# error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ + +#if SCUDO_TSD_EXCLUSIVE + +enum ThreadState : u8 { + ThreadNotInitialized = 0, + ThreadInitialized, + ThreadTornDown, +}; +__attribute__((tls_model("initial-exec"))) +extern THREADLOCAL ThreadState ScudoThreadState; +__attribute__((tls_model("initial-exec"))) +extern THREADLOCAL ScudoTSD TSD; + +extern ScudoTSD FallbackTSD; + +ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { + if (LIKELY(ScudoThreadState != ThreadNotInitialized)) + return; + initThread(MinimalInit); +} + +ALWAYS_INLINE ScudoTSD *getTSDAndLock() { + if (UNLIKELY(ScudoThreadState != ThreadInitialized)) { + FallbackTSD.lock(); + return &FallbackTSD; + } + return &TSD; +} + +#endif // SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp new file mode 100644 index 000000000000..3e13e5d3a109 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp @@ -0,0 +1,87 @@ +//===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo shared TSD implementation. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_tsd.h" + +#if !SCUDO_TSD_EXCLUSIVE + +namespace __scudo { + +static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; +pthread_key_t PThreadKey; + +static atomic_uint32_t CurrentIndex; +static ScudoTSD *TSDs; +static u32 NumberOfTSDs; + +static void initOnce() { + CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0); + initScudo(); + NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()), + static_cast<u32>(SCUDO_SHARED_TSD_POOL_SIZE)); + TSDs = reinterpret_cast<ScudoTSD *>( + MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs")); + for (u32 i = 0; i < NumberOfTSDs; i++) + TSDs[i].init(/*Shared=*/true); +} + +ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) { +#if SANITIZER_ANDROID + *get_android_tls_ptr() = reinterpret_cast<uptr>(TSD); +#else + CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(TSD)), 0); +#endif // SANITIZER_ANDROID +} + +void initThread(bool MinimalInit) { + pthread_once(&GlobalInitialized, initOnce); + // Initial context assignment is done in a plain round-robin fashion. + u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed); + setCurrentTSD(&TSDs[Index % NumberOfTSDs]); +} + +ScudoTSD *getTSDAndLockSlow() { + ScudoTSD *TSD; + if (NumberOfTSDs > 1) { + // Go through all the contexts and find the first unlocked one. + for (u32 i = 0; i < NumberOfTSDs; i++) { + TSD = &TSDs[i]; + if (TSD->tryLock()) { + setCurrentTSD(TSD); + return TSD; + } + } + // No luck, find the one with the lowest Precedence, and slow lock it. + u64 LowestPrecedence = UINT64_MAX; + for (u32 i = 0; i < NumberOfTSDs; i++) { + u64 Precedence = TSDs[i].getPrecedence(); + if (Precedence && Precedence < LowestPrecedence) { + TSD = &TSDs[i]; + LowestPrecedence = Precedence; + } + } + if (LIKELY(LowestPrecedence != UINT64_MAX)) { + TSD->lock(); + setCurrentTSD(TSD); + return TSD; + } + } + // Last resort, stick with the current one. + TSD = getCurrentTSD(); + TSD->lock(); + return TSD; +} + +} // namespace __scudo + +#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc new file mode 100644 index 000000000000..79fcd651ed2d --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc @@ -0,0 +1,48 @@ +//===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo shared TSD fastpath functions implementation. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +# error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ + +#if !SCUDO_TSD_EXCLUSIVE + +extern pthread_key_t PThreadKey; + +ALWAYS_INLINE ScudoTSD* getCurrentTSD() { +#if SANITIZER_ANDROID + return reinterpret_cast<ScudoTSD *>(*get_android_tls_ptr()); +#else + return reinterpret_cast<ScudoTSD *>(pthread_getspecific(PThreadKey)); +#endif // SANITIZER_ANDROID +} + +ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { + if (LIKELY(getCurrentTSD())) + return; + initThread(MinimalInit); +} + +ScudoTSD *getTSDAndLockSlow(); + +ALWAYS_INLINE ScudoTSD *getTSDAndLock() { + ScudoTSD *TSD = getCurrentTSD(); + CHECK(TSD && "No TSD associated with the current thread!"); + // Try to lock the currently associated context. + if (TSD->tryLock()) + return TSD; + // If it failed, go the slow path. + return getTSDAndLockSlow(); +} + +#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp index f7903ff34c73..2f936bf9e780 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp @@ -13,17 +13,18 @@ #include "scudo_utils.h" -#include <errno.h> -#include <fcntl.h> -#include <stdarg.h> -#include <unistd.h> #if defined(__x86_64__) || defined(__i386__) # include <cpuid.h> -#endif -#if defined(__arm__) || defined(__aarch64__) -# include <sys/auxv.h> +#elif defined(__arm__) || defined(__aarch64__) +# include "sanitizer_common/sanitizer_getauxval.h" +# if SANITIZER_POSIX +# include "sanitizer_common/sanitizer_posix.h" +# include <fcntl.h> +# endif #endif +#include <stdarg.h> + // TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less // complicated string formatting code. The following is a // temporary workaround to be able to use __sanitizer::VSNPrintf. @@ -36,13 +37,12 @@ extern int VSNPrintf(char *buff, int buff_length, const char *format, namespace __scudo { -FORMAT(1, 2) -void NORETURN dieWithMessage(const char *Format, ...) { +FORMAT(1, 2) void NORETURN dieWithMessage(const char *Format, ...) { // Our messages are tiny, 256 characters is more than enough. char Message[256]; va_list Args; va_start(Args, Format); - __sanitizer::VSNPrintf(Message, sizeof(Message), Format, Args); + VSNPrintf(Message, sizeof(Message), Format, Args); va_end(Args); RawWrite(Message); Die(); @@ -51,76 +51,68 @@ void NORETURN dieWithMessage(const char *Format, ...) { #if defined(__x86_64__) || defined(__i386__) // i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. // CRC32 requires the SSE 4.2 instruction set. -typedef struct { - u32 Eax; - u32 Ebx; - u32 Ecx; - u32 Edx; -} CPUIDRegs; - -static void getCPUID(CPUIDRegs *Regs, u32 Level) -{ - __get_cpuid(Level, &Regs->Eax, &Regs->Ebx, &Regs->Ecx, &Regs->Edx); +# ifndef bit_SSE4_2 +# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. +# endif +bool hasHardwareCRC32() { + u32 Eax, Ebx, Ecx, Edx; + __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx); + const bool IsIntel = (Ebx == signature_INTEL_ebx) && + (Edx == signature_INTEL_edx) && + (Ecx == signature_INTEL_ecx); + const bool IsAMD = (Ebx == signature_AMD_ebx) && + (Edx == signature_AMD_edx) && + (Ecx == signature_AMD_ecx); + if (!IsIntel && !IsAMD) + return false; + __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); + return !!(Ecx & bit_SSE4_2); } - -CPUIDRegs getCPUFeatures() { - CPUIDRegs VendorRegs = {}; - getCPUID(&VendorRegs, 0); - bool IsIntel = - (VendorRegs.Ebx == signature_INTEL_ebx) && - (VendorRegs.Edx == signature_INTEL_edx) && - (VendorRegs.Ecx == signature_INTEL_ecx); - bool IsAMD = - (VendorRegs.Ebx == signature_AMD_ebx) && - (VendorRegs.Edx == signature_AMD_edx) && - (VendorRegs.Ecx == signature_AMD_ecx); - // Default to an empty feature set if not on a supported CPU. - CPUIDRegs FeaturesRegs = {}; - if (IsIntel || IsAMD) { - getCPUID(&FeaturesRegs, 1); - } - return FeaturesRegs; -} - -#ifndef bit_SSE4_2 -# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. -#endif - -bool testCPUFeature(CPUFeature Feature) -{ - CPUIDRegs FeaturesRegs = getCPUFeatures(); - - switch (Feature) { - case CRC32CPUFeature: // CRC32 is provided by SSE 4.2. - return !!(FeaturesRegs.Ecx & bit_SSE4_2); - default: +#elif defined(__arm__) || defined(__aarch64__) +// For ARM and AArch64, hardware CRC32 support is indicated in the AT_HWCAP +// auxiliary vector. +# ifndef AT_HWCAP +# define AT_HWCAP 16 +# endif +# ifndef HWCAP_CRC32 +# define HWCAP_CRC32 (1 << 7) // HWCAP_CRC32 is missing on older platforms. +# endif +# if SANITIZER_POSIX +bool hasHardwareCRC32ARMPosix() { + uptr F = internal_open("/proc/self/auxv", O_RDONLY); + if (internal_iserror(F)) + return false; + struct { uptr Tag; uptr Value; } Entry = { 0, 0 }; + for (;;) { + uptr N = internal_read(F, &Entry, sizeof(Entry)); + if (internal_iserror(N) || N != sizeof(Entry) || + (Entry.Tag == 0 && Entry.Value == 0) || Entry.Tag == AT_HWCAP) break; } - return false; + internal_close(F); + return (Entry.Tag == AT_HWCAP && (Entry.Value & HWCAP_CRC32) != 0); } -#elif defined(__arm__) || defined(__aarch64__) -// For ARM and AArch64, hardware CRC32 support is indicated in the -// AT_HWVAL auxiliary vector. - -#ifndef HWCAP_CRC32 -# define HWCAP_CRC32 (1<<7) // HWCAP_CRC32 is missing on older platforms. -#endif +# else +bool hasHardwareCRC32ARMPosix() { return false; } +# endif // SANITIZER_POSIX -bool testCPUFeature(CPUFeature Feature) { - uptr HWCap = getauxval(AT_HWCAP); +// Bionic doesn't initialize its globals early enough. This causes issues when +// trying to access them from a preinit_array (b/25751302) or from another +// constructor called before the libc one (b/68046352). __progname is +// initialized after the other globals, so we can check its value to know if +// calling getauxval is safe. +extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname; +INLINE bool areBionicGlobalsInitialized() { + return !SANITIZER_ANDROID || (&__progname && __progname); +} - switch (Feature) { - case CRC32CPUFeature: - return !!(HWCap & HWCAP_CRC32); - default: - break; - } - return false; +bool hasHardwareCRC32() { + if (&getauxval && areBionicGlobalsInitialized()) + return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); + return hasHardwareCRC32ARMPosix(); } #else -bool testCPUFeature(CPUFeature Feature) { - return false; -} +bool hasHardwareCRC32() { return false; } #endif // defined(__x86_64__) || defined(__i386__) } // namespace __scudo diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.h b/contrib/compiler-rt/lib/scudo/scudo_utils.h index 6c6c9d893404..43448e0831e8 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_utils.h +++ b/contrib/compiler-rt/lib/scudo/scudo_utils.h @@ -14,14 +14,14 @@ #ifndef SCUDO_UTILS_H_ #define SCUDO_UTILS_H_ -#include <string.h> - #include "sanitizer_common/sanitizer_common.h" +#include <string.h> + namespace __scudo { template <class Dest, class Source> -inline Dest bit_cast(const Source& source) { +INLINE Dest bit_cast(const Source& source) { static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!"); Dest dest; memcpy(&dest, &source, sizeof(dest)); @@ -30,63 +30,7 @@ inline Dest bit_cast(const Source& source) { void NORETURN dieWithMessage(const char *Format, ...); -enum CPUFeature { - CRC32CPUFeature = 0, - MaxCPUFeature, -}; -bool testCPUFeature(CPUFeature feature); - -INLINE u64 rotl(const u64 X, int K) { - return (X << K) | (X >> (64 - K)); -} - -// XoRoShiRo128+ PRNG (http://xoroshiro.di.unimi.it/). -struct XoRoShiRo128Plus { - public: - void init() { - if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(State), sizeof(State)))) { - // Early processes (eg: init) do not have /dev/urandom yet, but we still - // have to provide them with some degree of entropy. Not having a secure - // seed is not as problematic for them, as they are less likely to be - // the target of heap based vulnerabilities exploitation attempts. - State[0] = NanoTime(); - State[1] = 0; - } - fillCache(); - } - u8 getU8() { - if (UNLIKELY(isCacheEmpty())) - fillCache(); - const u8 Result = static_cast<u8>(CachedBytes & 0xff); - CachedBytes >>= 8; - CachedBytesAvailable--; - return Result; - } - u64 getU64() { return next(); } - - private: - u8 CachedBytesAvailable; - u64 CachedBytes; - u64 State[2]; - u64 next() { - const u64 S0 = State[0]; - u64 S1 = State[1]; - const u64 Result = S0 + S1; - S1 ^= S0; - State[0] = rotl(S0, 55) ^ S1 ^ (S1 << 14); - State[1] = rotl(S1, 36); - return Result; - } - bool isCacheEmpty() { - return CachedBytesAvailable == 0; - } - void fillCache() { - CachedBytes = next(); - CachedBytesAvailable = sizeof(CachedBytes); - } -}; - -typedef XoRoShiRo128Plus ScudoPrng; +bool hasHardwareCRC32(); } // namespace __scudo diff --git a/contrib/compiler-rt/lib/stats/stats.cc b/contrib/compiler-rt/lib/stats/stats.cc index df9845a38de9..6a6eb3a3cdbc 100644 --- a/contrib/compiler-rt/lib/stats/stats.cc +++ b/contrib/compiler-rt/lib/stats/stats.cc @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_internal_defs.h" #if SANITIZER_POSIX #include "sanitizer_common/sanitizer_posix.h" diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc index d84df4a6413c..f13a7432ec95 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc @@ -48,8 +48,8 @@ static bool bogusfd(int fd) { } static FdSync *allocsync(ThreadState *thr, uptr pc) { - FdSync *s = (FdSync*)user_alloc(thr, pc, sizeof(FdSync), kDefaultAlignment, - false); + FdSync *s = (FdSync*)user_alloc_internal(thr, pc, sizeof(FdSync), + kDefaultAlignment, false); atomic_store(&s->rc, 1, memory_order_relaxed); return s; } @@ -79,7 +79,7 @@ static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) { if (l1 == 0) { uptr size = kTableSizeL2 * sizeof(FdDesc); // We need this to reside in user memory to properly catch races on it. - void *p = user_alloc(thr, pc, size, kDefaultAlignment, false); + void *p = user_alloc_internal(thr, pc, size, kDefaultAlignment, false); internal_memset(p, 0, size); MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size); if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel)) diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 001123f4941e..d92976ad6e9e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_posix.h" @@ -39,6 +40,21 @@ using namespace __tsan; // NOLINT #define stderr __stderrp #endif +#if SANITIZER_NETBSD +#define dirfd(dirp) (*(int *)(dirp)) +#define fileno_unlocked fileno + +#if _LP64 +#define __sF_size 152 +#else +#define __sF_size 88 +#endif + +#define stdout ((char*)&__sF + (__sF_size * 1)) +#define stderr ((char*)&__sF + (__sF_size * 2)) + +#endif + #if SANITIZER_ANDROID #define mallopt(a, b) #endif @@ -49,11 +65,6 @@ const int kSigCount = 129; const int kSigCount = 65; #endif -struct my_siginfo_t { - // The size is determined by looking at sizeof of real siginfo_t on linux. - u64 opaque[128 / sizeof(u64)]; -}; - #ifdef __mips__ struct ucontext_t { u64 opaque[768 / sizeof(u64) + 1]; @@ -84,19 +95,25 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int fileno_unlocked(void *stream); +#if !SANITIZER_NETBSD extern "C" int dirfd(void *dirp); -#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID +#endif +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD extern "C" int mallopt(int param, int value); #endif +#if SANITIZER_NETBSD +extern __sanitizer_FILE __sF[]; +#else extern __sanitizer_FILE *stdout, *stderr; -#if !SANITIZER_FREEBSD && !SANITIZER_MAC +#endif +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; #else const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif -#if !SANITIZER_FREEBSD && !SANITIZER_MAC +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD const int EPOLL_CTL_ADD = 1; #endif const int SIGILL = 4; @@ -105,7 +122,7 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC +#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; #else @@ -113,7 +130,9 @@ const int SIGBUS = 7; const int SIGSYS = 31; #endif void *const MAP_FAILED = (void*)-1; -#if !SANITIZER_MAC +#if SANITIZER_NETBSD +const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; +#elif !SANITIZER_MAC const int PTHREAD_BARRIER_SERIAL_THREAD = -1; #endif const int MAP_FIXED = 0x10; @@ -125,48 +144,7 @@ typedef long long_t; // NOLINT # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ -typedef void (*sighandler_t)(int sig); -typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); - -#if SANITIZER_ANDROID -struct sigaction_t { - u32 sa_flags; - union { - sighandler_t sa_handler; - sigactionhandler_t sa_sigaction; - }; - __sanitizer_sigset_t sa_mask; - void (*sa_restorer)(); -}; -#else -struct sigaction_t { -#ifdef __mips__ - u32 sa_flags; -#endif - union { - sighandler_t sa_handler; - sigactionhandler_t sa_sigaction; - }; -#if SANITIZER_FREEBSD - int sa_flags; - __sanitizer_sigset_t sa_mask; -#elif SANITIZER_MAC - __sanitizer_sigset_t sa_mask; - int sa_flags; -#else - __sanitizer_sigset_t sa_mask; -#ifndef __mips__ - int sa_flags; -#endif - void (*sa_restorer)(); -#endif -}; -#endif - -const sighandler_t SIG_DFL = (sighandler_t)0; -const sighandler_t SIG_IGN = (sighandler_t)1; -const sighandler_t SIG_ERR = (sighandler_t)-1; -#if SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) @@ -180,13 +158,11 @@ const int SIG_SETMASK = 2; #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ (!cur_thread()->is_inited) -static sigaction_t sigactions[kSigCount]; - namespace __tsan { struct SignalDesc { bool armed; bool sigaction; - my_siginfo_t siginfo; + __sanitizer_siginfo siginfo; ucontext_t ctx; }; @@ -200,11 +176,41 @@ struct ThreadSignalContext { __sanitizer_sigset_t oldset; }; -// The object is 64-byte aligned, because we want hot data to be located in -// a single cache line if possible (it's accessed in every interceptor). -static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)]; +// The sole reason tsan wraps atexit callbacks is to establish synchronization +// between callback setup and callback execution. +struct AtExitCtx { + void (*f)(); + void *arg; +}; + +// InterceptorContext holds all global data required for interceptors. +// It's explicitly constructed in InitializeInterceptors with placement new +// and is never destroyed. This allows usage of members with non-trivial +// constructors and destructors. +struct InterceptorContext { + // The object is 64-byte aligned, because we want hot data to be located + // in a single cache line if possible (it's accessed in every interceptor). + ALIGNED(64) LibIgnore libignore; + __sanitizer_sigaction sigactions[kSigCount]; +#if !SANITIZER_MAC && !SANITIZER_NETBSD + unsigned finalize_key; +#endif + + BlockingMutex atexit_mu; + Vector<struct AtExitCtx *> AtExitStack; + + InterceptorContext() + : libignore(LINKER_INITIALIZED), AtExitStack() { + } +}; + +static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; +InterceptorContext *interceptor_ctx() { + return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]); +} + LibIgnore *libignore() { - return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]); + return &interceptor_ctx()->libignore; } void InitializeLibIgnore() { @@ -232,10 +238,6 @@ static ThreadSignalContext *SigCtx(ThreadState *thr) { return ctx; } -#if !SANITIZER_MAC -static unsigned g_thread_finalize_key; -#endif - ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) { @@ -284,8 +286,18 @@ void ScopedInterceptor::DisableIgnores() { #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) #if SANITIZER_FREEBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) +#elif SANITIZER_NETBSD +# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ + INTERCEPT_FUNCTION(__libc_##func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ + INTERCEPT_FUNCTION(__libc_thr_##func) #else # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) #endif #define READ_STRING_OF_LEN(thr, pc, s, len, n) \ @@ -346,17 +358,30 @@ TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { return res; } -// The sole reason tsan wraps atexit callbacks is to establish synchronization -// between callback setup and callback execution. -struct AtExitCtx { - void (*f)(); - void *arg; -}; +TSAN_INTERCEPTOR(int, pause, int fake) { + SCOPED_TSAN_INTERCEPTOR(pause, fake); + return BLOCK_REAL(pause)(fake); +} -static void at_exit_wrapper(void *arg) { - ThreadState *thr = cur_thread(); - uptr pc = 0; - Acquire(thr, pc, (uptr)arg); +static void at_exit_wrapper() { + AtExitCtx *ctx; + { + // Ensure thread-safety. + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + // Pop AtExitCtx from the top of the stack of callback functions + uptr element = interceptor_ctx()->AtExitStack.Size() - 1; + ctx = interceptor_ctx()->AtExitStack[element]; + interceptor_ctx()->AtExitStack.PopBack(); + } + + Acquire(cur_thread(), (uptr)0, (uptr)ctx); + ((void(*)())ctx->f)(); + InternalFree(ctx); +} + +static void cxa_at_exit_wrapper(void *arg) { + Acquire(cur_thread(), 0, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(void *arg))ctx->f)(ctx->arg); InternalFree(ctx); @@ -367,7 +392,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. @@ -377,7 +402,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); @@ -392,12 +417,27 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. ThreadIgnoreBegin(thr, pc); - int res = REAL(__cxa_atexit)(at_exit_wrapper, ctx, dso); + int res; + if (!dso) { + // NetBSD does not preserve the 2nd argument if dso is equal to 0 + // Store ctx in a local stack-like structure + + // Ensure thread-safety. + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0); + // Push AtExitCtx on the top of the stack of callback functions + if (!res) { + interceptor_ctx()->AtExitStack.PushBack(ctx); + } + } else { + res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso); + } ThreadIgnoreEnd(thr, pc); return res; } -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_NETBSD static void on_exit_wrapper(int status, void *arg) { ThreadState *thr = cur_thread(); uptr pc = 0; @@ -408,7 +448,7 @@ static void on_exit_wrapper(int status, void *arg) { } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); @@ -422,6 +462,9 @@ TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { ThreadIgnoreEnd(thr, pc); return res; } +#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit) +#else +#define TSAN_MAYBE_INTERCEPT_ON_EXIT #endif // Cleanup old bufs. @@ -461,13 +504,15 @@ static void LongJmp(ThreadState *thr, uptr *env) { uptr mangled_sp = env[0]; #elif SANITIZER_FREEBSD uptr mangled_sp = env[2]; +#elif SANITIZER_NETBSD + uptr mangled_sp = env[6]; #elif SANITIZER_MAC # ifdef __aarch64__ uptr mangled_sp = env[13]; # else uptr mangled_sp = env[2]; # endif -#elif defined(SANITIZER_LINUX) +#elif SANITIZER_LINUX # ifdef __aarch64__ uptr mangled_sp = env[13]; # elif defined(__mips64) @@ -510,10 +555,29 @@ TSAN_INTERCEPTOR(int, setjmp, void *env); TSAN_INTERCEPTOR(int, _setjmp, void *env); TSAN_INTERCEPTOR(int, sigsetjmp, void *env); #else // SANITIZER_MAC + +#if SANITIZER_NETBSD +#define setjmp_symname __setjmp14 +#define sigsetjmp_symname __sigsetjmp14 +#else +#define setjmp_symname setjmp +#define sigsetjmp_symname sigsetjmp +#endif + +#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x +#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x) +#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname) +#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname) + +#define TSAN_STRING_SETJMP_(x) # x +#define TSAN_STRING_SETJMP__(x) TSAN_STRING_SETJMP_(x) +#define TSAN_STRING_SETJMP TSAN_STRING_SETJMP__(setjmp_symname) +#define TSAN_STRING_SIGSETJMP TSAN_STRING_SETJMP__(sigsetjmp_symname) + // Not called. Merely to satisfy TSAN_INTERCEPT(). extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int __interceptor_setjmp(void *env); -extern "C" int __interceptor_setjmp(void *env) { +int TSAN_INTERCEPTOR_SETJMP(void *env); +extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) { CHECK(0); return 0; } @@ -527,51 +591,75 @@ extern "C" int __interceptor__setjmp(void *env) { } extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int __interceptor_sigsetjmp(void *env); -extern "C" int __interceptor_sigsetjmp(void *env) { +int TSAN_INTERCEPTOR_SIGSETJMP(void *env); +extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) { CHECK(0); return 0; } +#if !SANITIEZER_NETBSD extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor___sigsetjmp(void *env); extern "C" int __interceptor___sigsetjmp(void *env) { CHECK(0); return 0; } +#endif -extern "C" int setjmp(void *env); +extern "C" int setjmp_symname(void *env); extern "C" int _setjmp(void *env); -extern "C" int sigsetjmp(void *env); +extern "C" int sigsetjmp_symname(void *env); +#if !SANITIEZER_NETBSD extern "C" int __sigsetjmp(void *env); -DEFINE_REAL(int, setjmp, void *env) +#endif +DEFINE_REAL(int, setjmp_symname, void *env) DEFINE_REAL(int, _setjmp, void *env) -DEFINE_REAL(int, sigsetjmp, void *env) +DEFINE_REAL(int, sigsetjmp_symname, void *env) +#if !SANITIEZER_NETBSD DEFINE_REAL(int, __sigsetjmp, void *env) +#endif #endif // SANITIZER_MAC -TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { +#if SANITIZER_NETBSD +#define longjmp_symname __longjmp14 +#define siglongjmp_symname __siglongjmp14 +#else +#define longjmp_symname longjmp +#define siglongjmp_symname siglongjmp +#endif + +TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) { // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, // bad things will happen. We will jump over ScopedInterceptor dtor and can // leave thr->in_ignored_lib set. { - SCOPED_INTERCEPTOR_RAW(longjmp, env, val); + SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val); } LongJmp(cur_thread(), env); - REAL(longjmp)(env, val); + REAL(longjmp_symname)(env, val); } -TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { +TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) { { - SCOPED_INTERCEPTOR_RAW(siglongjmp, env, val); + SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val); } LongJmp(cur_thread(), env); - REAL(siglongjmp)(env, val); + REAL(siglongjmp_symname)(env, val); } +#if SANITIZER_NETBSD +TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { + { + SCOPED_INTERCEPTOR_RAW(_longjmp, env, val); + } + LongJmp(cur_thread(), env); + REAL(_longjmp)(env, val); +} +#endif + #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalAlloc(size); void *p = 0; { @@ -584,11 +672,11 @@ TSAN_INTERCEPTOR(void*, malloc, uptr size) { TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); - return user_alloc(thr, pc, sz, align); + return user_memalign(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalCalloc(size, n); void *p = 0; { @@ -600,7 +688,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalRealloc(p, size); if (p) invoke_free_hook(p); @@ -615,7 +703,7 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); @@ -625,7 +713,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); @@ -730,7 +818,7 @@ TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(memalign, align, sz); - return user_alloc(thr, pc, sz, align); + return user_memalign(thr, pc, align, sz); } #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) #else @@ -739,21 +827,29 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { - SCOPED_INTERCEPTOR_RAW(memalign, align, sz); - return user_alloc(thr, pc, sz, align); + if (UNLIKELY(cur_thread()->in_symbolizer)) + return InternalAlloc(sz, nullptr, align); + SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); + return user_aligned_alloc(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { + if (UNLIKELY(cur_thread()->in_symbolizer)) + return InternalAlloc(sz, nullptr, GetPageSizeCached()); SCOPED_INTERCEPTOR_RAW(valloc, sz); - return user_alloc(thr, pc, sz, GetPageSizeCached()); + return user_valloc(thr, pc, sz); } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { + if (UNLIKELY(cur_thread()->in_symbolizer)) { + uptr PageSize = GetPageSizeCached(); + sz = sz ? RoundUpTo(sz, PageSize) : PageSize; + return InternalAlloc(sz, nullptr, PageSize); + } SCOPED_INTERCEPTOR_RAW(pvalloc, sz); - sz = RoundUp(sz, GetPageSizeCached()); - return user_alloc(thr, pc, sz, GetPageSizeCached()); + return user_pvalloc(thr, pc, sz); } #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) #else @@ -762,9 +858,15 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { + if (UNLIKELY(cur_thread()->in_symbolizer)) { + void *p = InternalAlloc(sz, nullptr, align); + if (!p) + return errno_ENOMEM; + *memptr = p; + return 0; + } SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); - *memptr = user_alloc(thr, pc, sz, align); - return 0; + return user_posix_memalign(thr, pc, memptr, align, sz); } #endif @@ -830,11 +932,12 @@ void DestroyThreadState() { } } // namespace __tsan -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_NETBSD static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { - if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { + if (pthread_setspecific(interceptor_ctx()->finalize_key, + (void*)(iter - 1))) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } @@ -860,9 +963,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) { ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_NETBSD ThreadIgnoreBegin(thr, 0); - if (pthread_setspecific(g_thread_finalize_key, + if (pthread_setspecific(interceptor_ctx()->finalize_key, (void *)GetPthreadDestructorIterations())) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); @@ -887,6 +990,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) { TSAN_INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); + + MaybeSpawnBackgroundThread(); + if (ctx->after_multithreaded_fork) { if (flags()->die_after_fork) { Report("ThreadSanitizer: starting new threads after multi-threaded " @@ -1312,10 +1418,15 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { if (o == 0 || f == 0) return errno_EINVAL; atomic_uint32_t *a; - if (!SANITIZER_MAC) - a = static_cast<atomic_uint32_t*>(o); - else // On OS X, pthread_once_t has a header with a long-sized signature. + + if (SANITIZER_MAC) a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t))); + else if (SANITIZER_NETBSD) + a = static_cast<atomic_uint32_t*> + ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz)); + else + a = static_cast<atomic_uint32_t*>(o); + u32 v = atomic_load(a, memory_order_acquire); if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { @@ -1347,7 +1458,7 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { #endif TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); @@ -1762,7 +1873,9 @@ TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, namespace __tsan { static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, - bool sigact, int sig, my_siginfo_t *info, void *uctx) { + bool sigact, int sig, + __sanitizer_siginfo *info, void *uctx) { + __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; if (acquire) Acquire(thr, 0, (uptr)&sigactions[sig]); // Signals are generally asynchronous, so if we receive a signals when @@ -1784,14 +1897,13 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, // This code races with sigaction. Be careful to not read sa_sigaction twice. // Also need to remember pc for reporting before the call, // because the handler can reset it. - volatile uptr pc = sigact ? - (uptr)sigactions[sig].sa_sigaction : - (uptr)sigactions[sig].sa_handler; - if (pc != (uptr)SIG_DFL && pc != (uptr)SIG_IGN) { + volatile uptr pc = + sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler; + if (pc != sig_dfl && pc != sig_ign) { if (sigact) - ((sigactionhandler_t)pc)(sig, info, uctx); + ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); else - ((sighandler_t)pc)(sig); + ((__sanitizer_sighandler_ptr)pc)(sig); } if (!ctx->after_multithreaded_fork) { thr->ignore_reads_and_writes = ignore_reads_and_writes; @@ -1855,7 +1967,8 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { } void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, - my_siginfo_t *info, void *ctx) { + __sanitizer_siginfo *info, + void *ctx) { ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { @@ -1905,58 +2018,10 @@ static void rtl_sighandler(int sig) { rtl_generic_sighandler(false, sig, 0, 0); } -static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { +static void rtl_sigaction(int sig, __sanitizer_siginfo *info, void *ctx) { rtl_generic_sighandler(true, sig, info, ctx); } -TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { - // Note: if we call REAL(sigaction) directly for any reason without proxying - // the signal handler through rtl_sigaction, very bad things will happen. - // The handler will run synchronously and corrupt tsan per-thread state. - SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); - if (old) - internal_memcpy(old, &sigactions[sig], sizeof(*old)); - if (act == 0) - return 0; - // Copy act into sigactions[sig]. - // Can't use struct copy, because compiler can emit call to memcpy. - // Can't use internal_memcpy, because it copies byte-by-byte, - // and signal handler reads the sa_handler concurrently. It it can read - // some bytes from old value and some bytes from new value. - // Use volatile to prevent insertion of memcpy. - sigactions[sig].sa_handler = *(volatile sighandler_t*)&act->sa_handler; - sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags; - internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, - sizeof(sigactions[sig].sa_mask)); -#if !SANITIZER_FREEBSD && !SANITIZER_MAC - sigactions[sig].sa_restorer = act->sa_restorer; -#endif - sigaction_t newact; - internal_memcpy(&newact, act, sizeof(newact)); - internal_sigfillset(&newact.sa_mask); - if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { - if (newact.sa_flags & SA_SIGINFO) - newact.sa_sigaction = rtl_sigaction; - else - newact.sa_handler = rtl_sighandler; - } - ReleaseStore(thr, pc, (uptr)&sigactions[sig]); - int res = REAL(sigaction)(sig, &newact, 0); - return res; -} - -TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { - sigaction_t act; - act.sa_handler = h; - internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); - act.sa_flags = 0; - sigaction_t old; - int res = sigaction(sig, &act, &old); - if (res) - return SIG_ERR; - return old.sa_handler; -} - TSAN_INTERCEPTOR(int, raise, int sig) { SCOPED_TSAN_INTERCEPTOR(raise, sig); ThreadSignalContext *sctx = SigCtx(thr); @@ -2020,7 +2085,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, } TSAN_INTERCEPTOR(int, fork, int fake) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); @@ -2270,6 +2335,77 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #include "sanitizer_common/sanitizer_common_interceptors.inc" +static int sigaction_impl(int sig, const __sanitizer_sigaction *act, + __sanitizer_sigaction *old); +static __sanitizer_sighandler_ptr signal_impl(int sig, + __sanitizer_sighandler_ptr h); + +#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ + { return sigaction_impl(signo, act, oldact); } + +#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ + { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); } + +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +int sigaction_impl(int sig, const __sanitizer_sigaction *act, + __sanitizer_sigaction *old) { + // Note: if we call REAL(sigaction) directly for any reason without proxying + // the signal handler through rtl_sigaction, very bad things will happen. + // The handler will run synchronously and corrupt tsan per-thread state. + SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); + __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; + __sanitizer_sigaction old_stored; + if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored)); + __sanitizer_sigaction newact; + if (act) { + // Copy act into sigactions[sig]. + // Can't use struct copy, because compiler can emit call to memcpy. + // Can't use internal_memcpy, because it copies byte-by-byte, + // and signal handler reads the handler concurrently. It it can read + // some bytes from old value and some bytes from new value. + // Use volatile to prevent insertion of memcpy. + sigactions[sig].handler = + *(volatile __sanitizer_sighandler_ptr const *)&act->handler; + sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags; + internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, + sizeof(sigactions[sig].sa_mask)); +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD + sigactions[sig].sa_restorer = act->sa_restorer; +#endif + internal_memcpy(&newact, act, sizeof(newact)); + internal_sigfillset(&newact.sa_mask); + if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) { + if (newact.sa_flags & SA_SIGINFO) + newact.sigaction = rtl_sigaction; + else + newact.handler = rtl_sighandler; + } + ReleaseStore(thr, pc, (uptr)&sigactions[sig]); + act = &newact; + } + int res = REAL(sigaction)(sig, act, old); + if (res == 0 && old) { + uptr cb = (uptr)old->sigaction; + if (cb == (uptr)rtl_sigaction || cb == (uptr)rtl_sighandler) { + internal_memcpy(old, &old_stored, sizeof(*old)); + } + } + return res; +} + +static __sanitizer_sighandler_ptr signal_impl(int sig, + __sanitizer_sighandler_ptr h) { + __sanitizer_sigaction act; + act.handler = h; + internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); + act.sa_flags = 0; + __sanitizer_sigaction old; + int res = sigaction_symname(sig, &act, &old); + if (res) return (__sanitizer_sighandler_ptr)sig_err; + return old.handler; +} + #define TSAN_SYSCALL() \ ThreadState *thr = cur_thread(); \ if (thr->ignore_interceptors) \ @@ -2290,7 +2426,7 @@ struct ScopedSyscall { } }; -#if !SANITIZER_FREEBSD && !SANITIZER_MAC +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { TSAN_SYSCALL(); MemoryAccessRange(thr, pc, p, s, write); @@ -2409,6 +2545,34 @@ TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { } #endif +#if SANITIZER_NETBSD +TSAN_INTERCEPTOR(void, _lwp_exit) { + SCOPED_TSAN_INTERCEPTOR(_lwp_exit); + DestroyThreadState(); + REAL(_lwp_exit)(); +} +#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit) +#else +#define TSAN_MAYBE_INTERCEPT__LWP_EXIT +#endif + +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()); + namespace __tsan { static void finalize(void *arg) { @@ -2440,20 +2604,30 @@ void InitializeInterceptors() { mallopt(-3, 32*1024); // M_MMAP_THRESHOLD #endif + new(interceptor_ctx()) InterceptorContext(); + InitializeCommonInterceptors(); + InitializeSignalInterceptors(); #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. using __interception::GetRealFunctionAddress; - GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0); + GetRealFunctionAddress(TSAN_STRING_SETJMP, + (uptr*)&REAL(setjmp_symname), 0, 0); GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); - GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0); + GetRealFunctionAddress(TSAN_STRING_SIGSETJMP, + (uptr*)&REAL(sigsetjmp_symname), 0, 0); +#if !SANITIZER_NETBSD GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif +#endif - TSAN_INTERCEPT(longjmp); - TSAN_INTERCEPT(siglongjmp); + TSAN_INTERCEPT(longjmp_symname); + TSAN_INTERCEPT(siglongjmp_symname); +#if SANITIZER_NETBSD + TSAN_INTERCEPT(_longjmp); +#endif TSAN_INTERCEPT(malloc); TSAN_INTERCEPT(__libc_memalign); @@ -2548,8 +2722,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(rmdir); TSAN_INTERCEPT(closedir); - TSAN_INTERCEPT(sigaction); - TSAN_INTERCEPT(signal); TSAN_INTERCEPT(sigsuspend); TSAN_INTERCEPT(sigblock); TSAN_INTERCEPT(sigsetmask); @@ -2560,6 +2732,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(sleep); TSAN_INTERCEPT(usleep); TSAN_INTERCEPT(nanosleep); + TSAN_INTERCEPT(pause); TSAN_INTERCEPT(gettimeofday); TSAN_INTERCEPT(getaddrinfo); @@ -2568,7 +2741,7 @@ void InitializeInterceptors() { #if !SANITIZER_ANDROID TSAN_INTERCEPT(dl_iterate_phdr); #endif - TSAN_INTERCEPT(on_exit); + TSAN_MAYBE_INTERCEPT_ON_EXIT; TSAN_INTERCEPT(__cxa_atexit); TSAN_INTERCEPT(_exit); @@ -2576,6 +2749,8 @@ void InitializeInterceptors() { TSAN_INTERCEPT(__tls_get_addr); #endif + TSAN_MAYBE_INTERCEPT__LWP_EXIT; + #if !SANITIZER_MAC && !SANITIZER_ANDROID // Need to setup it, because interceptors check that the function is resolved. // But atexit is emitted directly into the module, so can't be resolved. @@ -2587,13 +2762,30 @@ void InitializeInterceptors() { Die(); } -#if !SANITIZER_MAC - if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { +#if !SANITIZER_MAC && !SANITIZER_NETBSD + if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) { Printf("ThreadSanitizer: failed to create thread key\n"); Die(); } #endif + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); + FdInit(); } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h index de47466501da..959a39465e33 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h @@ -49,4 +49,16 @@ LibIgnore *libignore(); #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) +#if SANITIZER_NETBSD +# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \ + TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func)); +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \ + TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func)); +#else +# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) +#endif + #endif // TSAN_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc index 4f1079467331..3b3a4a2815c3 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -100,7 +100,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ return tsan_atomic_f##_compare_exchange_strong( \ - (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ kMacOrderNonBarrier, kMacOrderNonBarrier); \ } \ \ @@ -108,7 +108,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, t volatile *ptr) { \ SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ return tsan_atomic_f##_compare_exchange_strong( \ - (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ kMacOrderBarrier, kMacOrderNonBarrier); \ } @@ -122,14 +122,14 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, int64_t) -#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ - TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ - char *byte_ptr = ((char *)ptr) + (n >> 3); \ - char bit = 0x80u >> (n & 7); \ - char mask = clear ? ~bit : bit; \ - char orig_byte = op((a8 *)byte_ptr, mask, mo); \ - return orig_byte & bit; \ +#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ + TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ + volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ + char bit = 0x80u >> (n & 7); \ + char mask = clear ? ~bit : bit; \ + char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ + return orig_byte & bit; \ } #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc index f68a0468de53..a7922a42e42c 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc @@ -14,6 +14,7 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_vector.h" #include "tsan_interface_ann.h" #include "tsan_mutex.h" #include "tsan_report.h" @@ -21,7 +22,6 @@ #include "tsan_mman.h" #include "tsan_flags.h" #include "tsan_platform.h" -#include "tsan_vector.h" #define CALLERPC ((uptr)__builtin_return_address(0)) @@ -185,10 +185,10 @@ void PrintMatchedBenignRaces() { int unique_count = 0; int hit_count = 0; int add_count = 0; - Vector<ExpectRace> hit_matched(MBlockScopedBuf); + Vector<ExpectRace> hit_matched; CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count, &ExpectRace::hitcount); - Vector<ExpectRace> add_matched(MBlockScopedBuf); + Vector<ExpectRace> add_matched; CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count, &ExpectRace::addcount); if (hit_matched.Size()) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc index 8c759a3be4e1..eb22e4baa710 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc @@ -86,7 +86,8 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, void *orig_context, dispatch_function_t orig_work) { tsan_block_context_t *new_context = - (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t)); + (tsan_block_context_t *)user_alloc_internal(thr, pc, + sizeof(tsan_block_context_t)); new_context->queue = queue; new_context->orig_context = orig_context; new_context->orig_work = orig_work; @@ -176,7 +177,8 @@ static void invoke_and_release_block(void *param) { } #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ - TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ + DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ dispatch_block_t heap_block = Block_copy(block); \ @@ -266,7 +268,7 @@ TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, // need to undefine the macro. #undef dispatch_once TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, - dispatch_block_t block) { + DISPATCH_NOESCAPE dispatch_block_t block) { SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); u32 v = atomic_load(a, memory_order_acquire); @@ -476,7 +478,8 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, } TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, - dispatch_queue_t queue, void (^block)(size_t)) { + dispatch_queue_t queue, + DISPATCH_NOESCAPE void (^block)(size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); void *parent_to_child_sync = nullptr; @@ -519,9 +522,9 @@ TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, return REAL(dispatch_data_create)(buffer, size, q, destructor); if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) - destructor = ^(void) { WRAP(free)((void *)buffer); }; + destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) - destructor = ^(void) { WRAP(munmap)((void *)buffer, size); }; + destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(destructor); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc index 8d31ccbcaaca..455c95df6c50 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc @@ -26,7 +26,7 @@ using namespace __tsan; #define COMMON_MALLOC_FORCE_UNLOCK() #define COMMON_MALLOC_MEMALIGN(alignment, size) \ void *p = \ - user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment) + user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) #define COMMON_MALLOC_MALLOC(size) \ if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ SCOPED_INTERCEPTOR_RAW(malloc, size); \ @@ -43,7 +43,7 @@ using namespace __tsan; if (cur_thread()->in_symbolizer) \ return InternalAlloc(size, nullptr, GetPageSizeCached()); \ SCOPED_INTERCEPTOR_RAW(valloc, size); \ - void *p = user_alloc(thr, pc, size, GetPageSizeCached()) + void *p = user_valloc(thr, pc, size) #define COMMON_MALLOC_FREE(ptr) \ if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ SCOPED_INTERCEPTOR_RAW(free, ptr); \ diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc index f79dccddba9f..19680238bf76 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc @@ -13,6 +13,7 @@ #include "sanitizer_common/sanitizer_allocator_checks.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "tsan_mman.h" #include "tsan_rtl.h" @@ -149,11 +150,12 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) { OutputReport(thr, rep); } -void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { +void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align, + bool signal) { if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) return Allocator::FailureHandler::OnBadRequest(); void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align); - if (p == 0) + if (UNLIKELY(p == 0)) return 0; if (ctx && ctx->initialized) OnUserAlloc(thr, pc, (uptr)p, sz, true); @@ -162,15 +164,6 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { return p; } -void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { - if (CheckForCallocOverflow(size, n)) - return Allocator::FailureHandler::OnBadRequest(); - void *p = user_alloc(thr, pc, n * size); - if (p) - internal_memset(p, 0, n * size); - return p; -} - void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { ScopedGlobalProcessor sgp; if (ctx && ctx->initialized) @@ -180,6 +173,19 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { SignalUnsafeCall(thr, pc); } +void *user_alloc(ThreadState *thr, uptr pc, uptr sz) { + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment)); +} + +void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) + return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest()); + void *p = user_alloc_internal(thr, pc, n * size); + if (p) + internal_memset(p, 0, n * size); + return SetErrnoOnNull(p); +} + void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); ctx->metamap.AllocBlock(thr, pc, p, sz); @@ -200,15 +206,64 @@ void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) { void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { // FIXME: Handle "shrinking" more efficiently, // it seems that some software actually does this. - void *p2 = user_alloc(thr, pc, sz); - if (p2 == 0) - return 0; - if (p) { - uptr oldsz = user_alloc_usable_size(p); - internal_memcpy(p2, p, min(oldsz, sz)); + if (!p) + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz)); + if (!sz) { + user_free(thr, pc, p); + return nullptr; + } + void *new_p = user_alloc_internal(thr, pc, sz); + if (new_p) { + uptr old_sz = user_alloc_usable_size(p); + internal_memcpy(new_p, p, min(old_sz, sz)); user_free(thr, pc, p); } - return p2; + return SetErrnoOnNull(new_p); +} + +void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) { + if (UNLIKELY(!IsPowerOfTwo(align))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align)); +} + +int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, + uptr sz) { + if (UNLIKELY(!CheckPosixMemalignAlignment(align))) { + Allocator::FailureHandler::OnBadRequest(); + return errno_EINVAL; + } + void *ptr = user_alloc_internal(thr, pc, sz, align); + if (UNLIKELY(!ptr)) + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, align)); + *memptr = ptr; + return 0; +} + +void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align)); +} + +void *user_valloc(ThreadState *thr, uptr pc, uptr sz) { + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached())); +} + +void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) { + errno = errno_ENOMEM; + return Allocator::FailureHandler::OnBadRequest(); + } + // pvalloc(0) should allocate one page. + sz = sz ? RoundUpTo(sz, PageSize) : PageSize; + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize)); } uptr user_alloc_usable_size(const void *p) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h index 8cdeeb35aa6b..6042c5c5d96d 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h @@ -27,13 +27,20 @@ void AllocatorProcFinish(Processor *proc); void AllocatorPrintStats(); // For user allocations. -void *user_alloc(ThreadState *thr, uptr pc, uptr sz, - uptr align = kDefaultAlignment, bool signal = true); -void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); +void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, + uptr align = kDefaultAlignment, bool signal = true); // Does not accept NULL. void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); +// Interceptor implementations. +void *user_alloc(ThreadState *thr, uptr pc, uptr sz); +void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); -void *user_alloc_aligned(ThreadState *thr, uptr pc, uptr sz, uptr align); +void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz); +int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, + uptr sz); +void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz); +void *user_valloc(ThreadState *thr, uptr pc, uptr sz); +void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz); uptr user_alloc_usable_size(const void *p); // Invoking malloc/free hooks that may be installed by the user. diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h index bea1daba3952..bfac70df12d9 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -42,6 +42,19 @@ C/C++ on linux/x86_64 and freebsd/x86_64 7b00 0000 0000 - 7c00 0000 0000: heap 7c00 0000 0000 - 7e80 0000 0000: - 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack + +C/C++ on netbsd/amd64 can reuse the same mapping: + * The address space starts from 0x1000 (option with 0x0) and ends with + 0x7f7ffffff000. + * LoAppMem-kHeapMemEnd can be reused as it is. + * No VDSO support. + * No MidAppMem region. + * No additional HeapMem region. + * HiAppMem contains the stack, loader, shared libraries and heap. + * Stack on NetBSD/amd64 has prereserved 128MB. + * Heap grows downwards (top-down). + * ASLR must be disabled per-process or globally. + */ struct Mapping { static const uptr kMetaShadowBeg = 0x300000000000ull; @@ -111,7 +124,8 @@ C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) 0c00 0000 00 - 0d00 0000 00: - (4 GB) 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) 0e00 0000 00 - 0f00 0000 00: - (4 GB) -0f00 0000 00 - 1000 0000 00: traces (4 GB) +0f00 0000 00 - 0fc0 0000 00: traces (3 GB) +0fc0 0000 00 - 1000 0000 00: - */ struct Mapping { static const uptr kLoAppMemBeg = 0x0100000000ull; @@ -123,9 +137,9 @@ struct Mapping { static const uptr kMetaShadowBeg = 0x0d00000000ull; static const uptr kMetaShadowEnd = 0x0e00000000ull; static const uptr kTraceMemBeg = 0x0f00000000ull; - static const uptr kTraceMemEnd = 0x1000000000ull; - static const uptr kHiAppMemBeg = 0x1000000000ull; - static const uptr kHiAppMemEnd = 0x1000000000ull; + static const uptr kTraceMemEnd = 0x0fc0000000ull; + static const uptr kHiAppMemBeg = 0x0fc0000000ull; + static const uptr kHiAppMemEnd = 0x0fc0000000ull; static const uptr kAppMemMsk = 0x0ull; static const uptr kAppMemXor = 0x0ull; static const uptr kVdsoBeg = 0x7000000000000000ull; @@ -303,6 +317,38 @@ struct Mapping46 { static const uptr kVdsoBeg = 0x7800000000000000ull; }; +/* +C/C++ on linux/powerpc64 (47-bit VMA) +0000 0000 1000 - 0100 0000 0000: main binary +0100 0000 0000 - 0200 0000 0000: - +0100 0000 0000 - 1000 0000 0000: shadow +1000 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) +2000 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2200 0000 0000: traces +2200 0000 0000 - 7d00 0000 0000: - +7d00 0000 0000 - 7e00 0000 0000: heap +7e00 0000 0000 - 7e80 0000 0000: - +7e80 0000 0000 - 8000 0000 0000: modules and main thread stack +*/ +struct Mapping47 { + static const uptr kMetaShadowBeg = 0x100000000000ull; + static const uptr kMetaShadowEnd = 0x200000000000ull; + static const uptr kTraceMemBeg = 0x200000000000ull; + static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kHeapMemBeg = 0x7d0000000000ull; + static const uptr kHeapMemEnd = 0x7e0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x010000000000ull; + static const uptr kHiAppMemBeg = 0x7e8000000000ull; + static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits + static const uptr kAppMemMsk = 0x7c0000000000ull; + static const uptr kAppMemXor = 0x020000000000ull; + static const uptr kVdsoBeg = 0x7800000000000000ull; +}; + // Indicates the runtime will define the memory regions at runtime. #define TSAN_RUNTIME_VMA 1 #endif @@ -429,11 +475,13 @@ uptr MappingArchImpl(void) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return MappingImpl<Mapping44, Type>(); - else - return MappingImpl<Mapping46, Type>(); + switch (vmaSize) { + case 44: return MappingImpl<Mapping44, Type>(); + case 46: return MappingImpl<Mapping46, Type>(); + case 47: return MappingImpl<Mapping47, Type>(); + } DCHECK(0); + return 0; #else return MappingImpl<Mapping, Type>(); #endif @@ -582,11 +630,13 @@ bool IsAppMem(uptr mem) { DCHECK(0); return false; #elif defined(__powerpc64__) - if (vmaSize == 44) - return IsAppMemImpl<Mapping44>(mem); - else - return IsAppMemImpl<Mapping46>(mem); + switch (vmaSize) { + case 44: return IsAppMemImpl<Mapping44>(mem); + case 46: return IsAppMemImpl<Mapping46>(mem); + case 47: return IsAppMemImpl<Mapping47>(mem); + } DCHECK(0); + return false; #else return IsAppMemImpl<Mapping>(mem); #endif @@ -609,11 +659,13 @@ bool IsShadowMem(uptr mem) { DCHECK(0); return false; #elif defined(__powerpc64__) - if (vmaSize == 44) - return IsShadowMemImpl<Mapping44>(mem); - else - return IsShadowMemImpl<Mapping46>(mem); + switch (vmaSize) { + case 44: return IsShadowMemImpl<Mapping44>(mem); + case 46: return IsShadowMemImpl<Mapping46>(mem); + case 47: return IsShadowMemImpl<Mapping47>(mem); + } DCHECK(0); + return false; #else return IsShadowMemImpl<Mapping>(mem); #endif @@ -636,11 +688,13 @@ bool IsMetaMem(uptr mem) { DCHECK(0); return false; #elif defined(__powerpc64__) - if (vmaSize == 44) - return IsMetaMemImpl<Mapping44>(mem); - else - return IsMetaMemImpl<Mapping46>(mem); + switch (vmaSize) { + case 44: return IsMetaMemImpl<Mapping44>(mem); + case 46: return IsMetaMemImpl<Mapping46>(mem); + case 47: return IsMetaMemImpl<Mapping47>(mem); + } DCHECK(0); + return false; #else return IsMetaMemImpl<Mapping>(mem); #endif @@ -673,11 +727,13 @@ uptr MemToShadow(uptr x) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return MemToShadowImpl<Mapping44>(x); - else - return MemToShadowImpl<Mapping46>(x); + switch (vmaSize) { + case 44: return MemToShadowImpl<Mapping44>(x); + case 46: return MemToShadowImpl<Mapping46>(x); + case 47: return MemToShadowImpl<Mapping47>(x); + } DCHECK(0); + return 0; #else return MemToShadowImpl<Mapping>(x); #endif @@ -712,11 +768,13 @@ u32 *MemToMeta(uptr x) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return MemToMetaImpl<Mapping44>(x); - else - return MemToMetaImpl<Mapping46>(x); + switch (vmaSize) { + case 44: return MemToMetaImpl<Mapping44>(x); + case 46: return MemToMetaImpl<Mapping46>(x); + case 47: return MemToMetaImpl<Mapping47>(x); + } DCHECK(0); + return 0; #else return MemToMetaImpl<Mapping>(x); #endif @@ -764,11 +822,13 @@ uptr ShadowToMem(uptr s) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return ShadowToMemImpl<Mapping44>(s); - else - return ShadowToMemImpl<Mapping46>(s); + switch (vmaSize) { + case 44: return ShadowToMemImpl<Mapping44>(s); + case 46: return ShadowToMemImpl<Mapping46>(s); + case 47: return ShadowToMemImpl<Mapping47>(s); + } DCHECK(0); + return 0; #else return ShadowToMemImpl<Mapping>(s); #endif @@ -799,11 +859,13 @@ uptr GetThreadTrace(int tid) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return GetThreadTraceImpl<Mapping44>(tid); - else - return GetThreadTraceImpl<Mapping46>(tid); + switch (vmaSize) { + case 44: return GetThreadTraceImpl<Mapping44>(tid); + case 46: return GetThreadTraceImpl<Mapping46>(tid); + case 47: return GetThreadTraceImpl<Mapping47>(tid); + } DCHECK(0); + return 0; #else return GetThreadTraceImpl<Mapping>(tid); #endif @@ -829,11 +891,13 @@ uptr GetThreadTraceHeader(int tid) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return GetThreadTraceHeaderImpl<Mapping44>(tid); - else - return GetThreadTraceHeaderImpl<Mapping46>(tid); + switch (vmaSize) { + case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid); + case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid); + case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid); + } DCHECK(0); + return 0; #else return GetThreadTraceHeaderImpl<Mapping>(tid); #endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc index ead1e5704989..e14d5f575a2e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc @@ -14,11 +14,12 @@ #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_procmaps.h" @@ -216,9 +217,9 @@ void InitializePlatformEarly() { Die(); } #elif defined(__powerpc64__) - if (vmaSize != 44 && vmaSize != 46) { + if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); - Printf("FATAL: Found %d - Supported 44 and 46\n", vmaSize); + Printf("FATAL: Found %d - Supported 44, 46, and 47\n", vmaSize); Die(); } #endif @@ -401,4 +402,4 @@ void cur_thread_finalize() { } // namespace __tsan -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc index 73a656ffca5e..f8d7324b763c 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc @@ -167,8 +167,8 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { #else // !SANITIZER_GO "app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" #endif - "stacks: %ld unique IDs, %ld kB allocated\n" - "threads: %ld total, %ld live\n" + "stacks: %zd unique IDs, %zd kB allocated\n" + "threads: %zd total, %zd live\n" "------------------------------\n", ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024, MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024, @@ -231,7 +231,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, void InitializePlatformEarly() { #if defined(__aarch64__) - uptr max_vm = GetMaxVirtualAddress() + 1; + uptr max_vm = GetMaxUserVirtualAddress() + 1; if (max_vm != Mapping::kHiAppMemEnd) { Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n", max_vm, Mapping::kHiAppMemEnd); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc index 32cc3325fe40..af47076968d3 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc @@ -13,6 +13,7 @@ #include "tsan_report.h" #include "tsan_platform.h" #include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stacktrace_printer.h" @@ -38,34 +39,27 @@ ReportLocation *ReportLocation::New(ReportLocationType type) { class Decorator: public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() { } - const char *Warning() { return Red(); } - const char *EndWarning() { return Default(); } const char *Access() { return Blue(); } - const char *EndAccess() { return Default(); } const char *ThreadDescription() { return Cyan(); } - const char *EndThreadDescription() { return Default(); } const char *Location() { return Green(); } - const char *EndLocation() { return Default(); } const char *Sleep() { return Yellow(); } - const char *EndSleep() { return Default(); } const char *Mutex() { return Magenta(); } - const char *EndMutex() { return Default(); } }; ReportDesc::ReportDesc() : tag(kExternalTagNone) - , stacks(MBlockReportStack) - , mops(MBlockReportMop) - , locs(MBlockReportLoc) - , mutexes(MBlockReportMutex) - , threads(MBlockReportThread) - , unique_tids(MBlockReportThread) + , stacks() + , mops() + , locs() + , mutexes() + , threads() + , unique_tids() , sleep() , count() { } ReportMop::ReportMop() - : mset(MBlockReportMutex) { + : mset() { } ReportDesc::~ReportDesc() { @@ -180,7 +174,7 @@ static void PrintMop(const ReportMop *mop, bool first) { } PrintMutexSet(mop->mset); Printf(":\n"); - Printf("%s", d.EndAccess()); + Printf("%s", d.Default()); PrintStack(mop->stack); } @@ -221,20 +215,20 @@ static void PrintLocation(const ReportLocation *loc) { loc->fd, thread_name(thrbuf, loc->tid)); print_stack = true; } - Printf("%s", d.EndLocation()); + Printf("%s", d.Default()); if (print_stack) PrintStack(loc->stack); } static void PrintMutexShort(const ReportMutex *rm, const char *after) { Decorator d; - Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.EndMutex(), after); + Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after); } static void PrintMutexShortWithAddress(const ReportMutex *rm, const char *after) { Decorator d; - Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.EndMutex(), after); + Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after); } static void PrintMutex(const ReportMutex *rm) { @@ -242,11 +236,11 @@ static void PrintMutex(const ReportMutex *rm) { if (rm->destroyed) { Printf("%s", d.Mutex()); Printf(" Mutex M%llu is already destroyed.\n\n", rm->id); - Printf("%s", d.EndMutex()); + Printf("%s", d.Default()); } else { Printf("%s", d.Mutex()); Printf(" Mutex M%llu (%p) created at:\n", rm->id, rm->addr); - Printf("%s", d.EndMutex()); + Printf("%s", d.Default()); PrintStack(rm->stack); } } @@ -264,7 +258,7 @@ static void PrintThread(const ReportThread *rt) { if (rt->workerthread) { Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status); Printf("\n"); - Printf("%s", d.EndThreadDescription()); + Printf("%s", d.Default()); return; } Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status, @@ -272,7 +266,7 @@ static void PrintThread(const ReportThread *rt) { if (rt->stack) Printf(" at:"); Printf("\n"); - Printf("%s", d.EndThreadDescription()); + Printf("%s", d.Default()); PrintStack(rt->stack); } @@ -280,7 +274,7 @@ static void PrintSleep(const ReportStack *s) { Decorator d; Printf("%s", d.Sleep()); Printf(" As if synchronized via sleep:\n"); - Printf("%s", d.EndSleep()); + Printf("%s", d.Default()); PrintStack(s); } @@ -324,7 +318,7 @@ void PrintReport(const ReportDesc *rep) { Printf("%s", d.Warning()); Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str, (int)internal_getpid()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); if (rep->typ == ReportTypeDeadlock) { char thrbuf[kThreadBufSize]; @@ -342,7 +336,7 @@ void PrintReport(const ReportDesc *rep) { PrintMutexShort(rep->mutexes[i], " in "); Printf("%s", d.ThreadDescription()); Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i])); - Printf("%s", d.EndThreadDescription()); + Printf("%s", d.Default()); if (flags()->second_deadlock_stack) { PrintStack(rep->stacks[2*i]); Printf(" Mutex "); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h index bc1582f90fe4..cdc999c6af1d 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h @@ -14,8 +14,8 @@ #define TSAN_REPORT_H #include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_vector.h" #include "tsan_defs.h" -#include "tsan_vector.h" namespace __tsan { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc index a01525302b02..17b820977c26 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc @@ -14,6 +14,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_placement_new.h" @@ -101,8 +102,8 @@ Context::Context() , thread_registry(new(thread_registry_placeholder) ThreadRegistry( CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)) , racy_mtx(MutexTypeRacy, StatMtxRacy) - , racy_stacks(MBlockRacyStacks) - , racy_addresses(MBlockRacyAddresses) + , racy_stacks() + , racy_addresses() , fired_suppressions_mtx(MutexTypeFired, StatMtxFired) , fired_suppressions(8) , clock_alloc("clock allocator") { @@ -120,7 +121,7 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, // , ignore_interceptors() , clock(tid, reuse_count) #if !SANITIZER_GO - , jmp_bufs(MBlockJmpBuf) + , jmp_bufs() #endif , tid(tid) , unique_id(unique_id) @@ -213,7 +214,7 @@ static void BackgroundThread(void *arg) { memory_order_relaxed); if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) { Lock l(&ctx->report_mtx); - SpinMutexLock l2(&CommonSanitizerReportMutex); + ScopedErrorReportLock l2; SymbolizeFlush(); atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed); } @@ -323,6 +324,21 @@ static void CheckShadowMapping() { } } +#if !SANITIZER_GO +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + uptr top = 0; + uptr bottom = 0; + bool fast = common_flags()->fast_unwind_on_fatal; + if (fast) GetThreadStackTopAndBottom(false, &top, &bottom); + stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast); +} + +static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr); +} +#endif + void Initialize(ThreadState *thr) { // Thread safe because done before all threads exist. static bool is_initialized = false; @@ -360,6 +376,7 @@ void Initialize(ThreadState *thr) { #if !SANITIZER_GO InitializeShadowMemory(); InitializeAllocatorLate(); + InstallDeadlySignalHandlers(TsanOnDeadlySignal); #endif // Setup correct file descriptor for error reports. __sanitizer_set_report_path(common_flags()->log_path); @@ -367,13 +384,6 @@ void Initialize(ThreadState *thr) { #if !SANITIZER_GO InitializeLibIgnore(); Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); - // On MIPS, TSan initialization is run before - // __pthread_initialize_minimal_internal() is finished, so we can not spawn - // new threads. -#ifndef __mips__ - StartBackgroundThread(); - SetSandboxingCallback(StopBackgroundThread); -#endif #endif VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n", @@ -402,6 +412,21 @@ void Initialize(ThreadState *thr) { OnInitialize(); } +void MaybeSpawnBackgroundThread() { + // On MIPS, TSan initialization is run before + // __pthread_initialize_minimal_internal() is finished, so we can not spawn + // new threads. +#if !SANITIZER_GO && !defined(__mips__) + static atomic_uint32_t bg_thread = {}; + if (atomic_load(&bg_thread, memory_order_relaxed) == 0 && + atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) { + StartBackgroundThread(); + SetSandboxingCallback(StopBackgroundThread); + } +#endif +} + + int Finalize(ThreadState *thr) { bool failed = false; @@ -412,8 +437,7 @@ int Finalize(ThreadState *thr) { // Wait for pending reports. ctx->report_mtx.Lock(); - CommonSanitizerReportMutex.Lock(); - CommonSanitizerReportMutex.Unlock(); + { ScopedErrorReportLock l; } ctx->report_mtx.Unlock(); #if !SANITIZER_GO diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h index 2cf2e168454d..7d44ccae9893 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h @@ -34,12 +34,13 @@ #include "sanitizer_common/sanitizer_libignore.h" #include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_thread_registry.h" +#include "sanitizer_common/sanitizer_vector.h" #include "tsan_clock.h" #include "tsan_defs.h" #include "tsan_flags.h" +#include "tsan_mman.h" #include "tsan_sync.h" #include "tsan_trace.h" -#include "tsan_vector.h" #include "tsan_report.h" #include "tsan_platform.h" #include "tsan_mutexset.h" @@ -574,11 +575,8 @@ const char *GetObjectTypeFromTag(uptr tag); const char *GetReportHeaderFromTag(uptr tag); uptr TagFromShadowStackFrame(uptr pc); -class ScopedReport { +class ScopedReportBase { public: - explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone); - ~ScopedReport(); - void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack, const MutexSet *mset); void AddStack(StackTrace stack, bool suppressable = false); @@ -593,6 +591,10 @@ class ScopedReport { const ReportDesc *GetReport() const; + protected: + ScopedReportBase(ReportType typ, uptr tag); + ~ScopedReportBase(); + private: ReportDesc *rep_; // Symbolizer makes lots of intercepted calls. If we try to process them, @@ -601,8 +603,17 @@ class ScopedReport { void AddDeadMutex(u64 id); - ScopedReport(const ScopedReport&); - void operator = (const ScopedReport&); + ScopedReportBase(const ScopedReportBase &) = delete; + void operator=(const ScopedReportBase &) = delete; +}; + +class ScopedReport : public ScopedReportBase { + public: + explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone); + ~ScopedReport(); + + private: + ScopedErrorReportLock lock_; }; ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack); @@ -690,6 +701,7 @@ void PrintCurrentStack(ThreadState *thr, uptr pc); void PrintCurrentStackSlow(uptr pc); // uses libunwind void Initialize(ThreadState *thr); +void MaybeSpawnBackgroundThread(); int Finalize(ThreadState *thr); void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); @@ -825,7 +837,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, return; DCHECK_GE((int)typ, 0); DCHECK_LE((int)typ, 7); - DCHECK_EQ(GetLsb(addr, 61), addr); + DCHECK_EQ(GetLsb(addr, kEventPCBits), addr); StatInc(thr, StatEvents); u64 pos = fs.GetTracePos(); if (UNLIKELY((pos % kTracePartSize) == 0)) { @@ -837,7 +849,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, } Event *trace = (Event*)GetThreadTrace(fs.tid()); Event *evp = &trace[pos]; - Event ev = (u64)addr | ((u64)typ << 61); + Event ev = (u64)addr | ((u64)typ << kEventPCBits); *evp = ev; } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S index f8c6453a5da4..f1e4dbb50fbb 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -6,7 +6,7 @@ #if !defined(__APPLE__) .section .bss .type __tsan_pointer_chk_guard, %object -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) __tsan_pointer_chk_guard: .zero 8 #endif @@ -51,7 +51,7 @@ _sigsetjmp$non_lazy_ptr: // original ones. ASM_HIDDEN(_Z18InitializeGuardPtrv) .global _Z18InitializeGuardPtrv -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) _Z18InitializeGuardPtrv: CFI_STARTPROC // Allocates a jmp_buf for the setjmp call. @@ -88,14 +88,14 @@ _Z18InitializeGuardPtrv: CFI_DEF_CFA (31, 0) ret CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) #endif ASM_HIDDEN(__tsan_setjmp) .comm _ZN14__interception11real_setjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): +.globl ASM_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC // save env parameters for function call @@ -125,7 +125,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) // restore env parameter mov x0, x19 @@ -148,12 +148,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): br x1 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) .comm _ZN14__interception12real__setjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): +.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameters for function call @@ -183,7 +183,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) // Restore jmp_buf parameter mov x0, x19 @@ -206,12 +206,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): br x1 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -243,7 +243,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) // restore env parameter mov w1, w20 @@ -268,13 +268,13 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): #endif br x2 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) #if !defined(__APPLE__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -303,7 +303,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) mov w1, w20 mov x0, x19 @@ -321,12 +321,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE] ldr x2, [x2] #else - adrp x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page - add x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff + adrp x2, ASM_SYMBOL(__sigsetjmp)@page + add x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff #endif br x2 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif #if defined(__FreeBSD__) || defined(__linux__) diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S index 98947fd2a1ba..8af61bf0e892 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S @@ -10,8 +10,8 @@ #endif ASM_HIDDEN(__tsan_trace_switch) -.globl ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk) -ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): +.globl ASM_SYMBOL(__tsan_trace_switch_thunk) +ASM_SYMBOL(__tsan_trace_switch_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -50,7 +50,7 @@ ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call ASM_TSAN_SYMBOL(__tsan_trace_switch) + call ASM_SYMBOL(__tsan_trace_switch) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -90,8 +90,8 @@ ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): CFI_ENDPROC ASM_HIDDEN(__tsan_report_race) -.globl ASM_TSAN_SYMBOL(__tsan_report_race_thunk) -ASM_TSAN_SYMBOL(__tsan_report_race_thunk): +.globl ASM_SYMBOL(__tsan_report_race_thunk) +ASM_SYMBOL(__tsan_report_race_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -130,7 +130,7 @@ ASM_TSAN_SYMBOL(__tsan_report_race_thunk): shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call ASM_TSAN_SYMBOL(__tsan_report_race) + call ASM_SYMBOL(__tsan_report_race) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -170,19 +170,27 @@ ASM_TSAN_SYMBOL(__tsan_report_race_thunk): CFI_ENDPROC ASM_HIDDEN(__tsan_setjmp) -#if !defined(__APPLE__) +#if defined(__NetBSD__) +.comm _ZN14__interception15real___setjmp14E,8,8 +#elif !defined(__APPLE__) .comm _ZN14__interception11real_setjmpE,8,8 #endif -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): +#if defined(__NetBSD__) +.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) +ASM_SYMBOL_INTERCEPTOR(__setjmp14): +#else +.globl ASM_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SYMBOL_INTERCEPTOR(setjmp): +#endif CFI_STARTPROC // save env parameter push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) // obtain %rsp -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi #elif defined(__APPLE__) @@ -197,33 +205,40 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): # error "Unknown platform" #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) CFI_RESTORE(%rdi) // tail jump to libc setjmp movl $0, %eax -#if !defined(__APPLE__) +#if defined(__NetBSD__) + movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#elif !defined(__APPLE__) movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) #else - jmp ASM_TSAN_SYMBOL(setjmp) + jmp ASM_SYMBOL(setjmp) #endif CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +#if defined(__NetBSD__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) +#else +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) +#endif .comm _ZN14__interception12real__setjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): +.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameter push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) // obtain %rsp -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi #elif defined(__APPLE__) @@ -238,7 +253,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): # error "Unknown platform" #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) @@ -249,15 +264,22 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) #else - jmp ASM_TSAN_SYMBOL(_setjmp) + jmp ASM_SYMBOL(_setjmp) #endif CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +#if defined(__NetBSD__) +.comm _ZN14__interception18real___sigsetjmp14E,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14): +#else .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(sigsetjmp): +#endif CFI_STARTPROC // save env parameter push %rdi @@ -271,7 +293,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): sub $8, %rsp CFI_ADJUST_CFA_OFFSET(8) // obtain %rsp -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) lea 24(%rsp), %rdi mov %rdi, %rsi #elif defined(__APPLE__) @@ -286,7 +308,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): # error "Unknown platform" #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -300,20 +322,27 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_RESTORE(%rdi) // tail jump to libc sigsetjmp movl $0, %eax -#if !defined(__APPLE__) +#if defined(__NetBSD__) + movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#elif !defined(__APPLE__) movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) #else - jmp ASM_TSAN_SYMBOL(sigsetjmp) + jmp ASM_SYMBOL(sigsetjmp) #endif CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +#if defined(__NetBSD__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) +#else +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +#endif -#if !defined(__APPLE__) +#if !defined(__APPLE__) && !defined(__NetBSD__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameter push %rdi @@ -337,7 +366,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): rol $0x11, %rsi #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -354,11 +383,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) -#endif // !defined(__APPLE__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif // !defined(__APPLE__) && !defined(__NetBSD__) #if defined(__FreeBSD__) || defined(__linux__) /* We do not need executable stack. */ +/* This note is not needed on NetBSD. */ .section .note.GNU-stack,"",@progbits #endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc index 2f85811620f1..152b965ad535 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -84,7 +84,9 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true); if (s == 0) return; - if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit)) { + if ((flagz & MutexFlagLinkerInit) + || s->IsFlagSet(MutexFlagLinkerInit) + || ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) { // Destroy is no-op for linker-initialized mutexes. s->mtx.Unlock(); return; diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc index 85a982941ed7..cc582ab50190 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc @@ -143,30 +143,28 @@ static ReportStack *SymbolizeStack(StackTrace trace) { return stack; } -ScopedReport::ScopedReport(ReportType typ, uptr tag) { +ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) { ctx->thread_registry->CheckLocked(); void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc)); rep_ = new(mem) ReportDesc; rep_->typ = typ; rep_->tag = tag; ctx->report_mtx.Lock(); - CommonSanitizerReportMutex.Lock(); } -ScopedReport::~ScopedReport() { - CommonSanitizerReportMutex.Unlock(); +ScopedReportBase::~ScopedReportBase() { ctx->report_mtx.Unlock(); DestroyAndFree(rep_); } -void ScopedReport::AddStack(StackTrace stack, bool suppressable) { +void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) { ReportStack **rs = rep_->stacks.PushBack(); *rs = SymbolizeStack(stack); (*rs)->suppressable = suppressable; } -void ScopedReport::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, - StackTrace stack, const MutexSet *mset) { +void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, + StackTrace stack, const MutexSet *mset) { void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop)); ReportMop *mop = new(mem) ReportMop; rep_->mops.PushBack(mop); @@ -187,11 +185,11 @@ void ScopedReport::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, } } -void ScopedReport::AddUniqueTid(int unique_tid) { +void ScopedReportBase::AddUniqueTid(int unique_tid) { rep_->unique_tids.PushBack(unique_tid); } -void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) { +void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { for (uptr i = 0; i < rep_->threads.Size(); i++) { if ((u32)rep_->threads[i]->id == tctx->tid) return; @@ -255,14 +253,14 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { } #endif -void ScopedReport::AddThread(int unique_tid, bool suppressable) { +void ScopedReportBase::AddThread(int unique_tid, bool suppressable) { #if !SANITIZER_GO if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid)) AddThread(tctx, suppressable); #endif } -void ScopedReport::AddMutex(const SyncVar *s) { +void ScopedReportBase::AddMutex(const SyncVar *s) { for (uptr i = 0; i < rep_->mutexes.Size(); i++) { if (rep_->mutexes[i]->id == s->uid) return; @@ -276,7 +274,7 @@ void ScopedReport::AddMutex(const SyncVar *s) { rm->stack = SymbolizeStackId(s->creation_stack_id); } -u64 ScopedReport::AddMutex(u64 id) { +u64 ScopedReportBase::AddMutex(u64 id) { u64 uid = 0; u64 mid = id; uptr addr = SyncVar::SplitId(id, &uid); @@ -295,7 +293,7 @@ u64 ScopedReport::AddMutex(u64 id) { return mid; } -void ScopedReport::AddDeadMutex(u64 id) { +void ScopedReportBase::AddDeadMutex(u64 id) { for (uptr i = 0; i < rep_->mutexes.Size(); i++) { if (rep_->mutexes[i]->id == id) return; @@ -309,7 +307,7 @@ void ScopedReport::AddDeadMutex(u64 id) { rm->stack = 0; } -void ScopedReport::AddLocation(uptr addr, uptr size) { +void ScopedReportBase::AddLocation(uptr addr, uptr size) { if (addr == 0) return; #if !SANITIZER_GO @@ -364,18 +362,19 @@ void ScopedReport::AddLocation(uptr addr, uptr size) { } #if !SANITIZER_GO -void ScopedReport::AddSleep(u32 stack_id) { +void ScopedReportBase::AddSleep(u32 stack_id) { rep_->sleep = SymbolizeStackId(stack_id); } #endif -void ScopedReport::SetCount(int count) { - rep_->count = count; -} +void ScopedReportBase::SetCount(int count) { rep_->count = count; } -const ReportDesc *ScopedReport::GetReport() const { - return rep_; -} +const ReportDesc *ScopedReportBase::GetReport() const { return rep_; } + +ScopedReport::ScopedReport(ReportType typ, uptr tag) + : ScopedReportBase(typ, tag) {} + +ScopedReport::~ScopedReport() {} void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, MutexSet *mset, uptr *tag) { @@ -394,7 +393,7 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, const u64 ebegin = RoundDown(eend, kTracePartSize); DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n", tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx); - Vector<uptr> stack(MBlockReportStack); + Vector<uptr> stack; stack.Resize(hdr->stack0.size + 64); for (uptr i = 0; i < hdr->stack0.size; i++) { stack[i] = hdr->stack0.trace[i]; @@ -406,8 +405,8 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, Event *events = (Event*)GetThreadTrace(tid); for (uptr i = ebegin; i <= eend; i++) { Event ev = events[i]; - EventType typ = (EventType)(ev >> 61); - uptr pc = (uptr)(ev & ((1ull << 61) - 1)); + EventType typ = (EventType)(ev >> kEventPCBits); + uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1)); DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc); if (typ == EventTypeMop) { stack[pos] = pc; @@ -634,13 +633,33 @@ void ReportRace(ThreadState *thr) { const uptr kMop = 2; VarSizeStackTrace traces[kMop]; uptr tags[kMop] = {kExternalTagNone}; - const uptr toppc = TraceTopPC(thr); + uptr toppc = TraceTopPC(thr); + if (toppc >> kEventPCBits) { + // This is a work-around for a known issue. + // The scenario where this happens is rather elaborate and requires + // an instrumented __sanitizer_report_error_summary callback and + // a __tsan_symbolize_external callback and a race during a range memory + // access larger than 8 bytes. MemoryAccessRange adds the current PC to + // the trace and starts processing memory accesses. A first memory access + // triggers a race, we report it and call the instrumented + // __sanitizer_report_error_summary, which adds more stuff to the trace + // since it is intrumented. Then a second memory access in MemoryAccessRange + // also triggers a race and we get here and call TraceTopPC to get the + // current PC, however now it contains some unrelated events from the + // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit + // event. Later we subtract -1 from it (in GetPreviousInstructionPc) + // and the resulting PC has kExternalPCBit set, so we pass it to + // __tsan_symbolize_external. __tsan_symbolize_external is within its rights + // to crash since the PC is completely bogus. + // test/tsan/double_race.cc contains a test case for this. + toppc = 0; + } ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]); if (IsFiredSuppression(ctx, typ, traces[0])) return; // MutexSet is too large to live on stack. - Vector<u64> mset_buffer(MBlockScopedBuf); + Vector<u64> mset_buffer; mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1); MutexSet *mset2 = new(&mset_buffer[0]) MutexSet(); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc index 83fab082afe3..e4d65b9a909b 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc @@ -211,7 +211,7 @@ void ThreadFinalize(ThreadState *thr) { if (!flags()->report_thread_leaks) return; ThreadRegistryLock l(ctx->thread_registry); - Vector<ThreadLeak> leaks(MBlockScopedBuf); + Vector<ThreadLeak> leaks; ctx->thread_registry->RunCallbackForEachThreadLocked( MaybeReportThreadLeak, &leaks); for (uptr i = 0; i < leaks.Size(); i++) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h index b83c09ff784e..9039970bcf86 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h @@ -34,6 +34,7 @@ enum MutexFlags { MutexFlagTryLockFailed = 1 << 5, // __tsan_mutex_try_lock_failed MutexFlagRecursiveLock = 1 << 6, // __tsan_mutex_recursive_lock MutexFlagRecursiveUnlock = 1 << 7, // __tsan_mutex_recursive_unlock + MutexFlagNotStatic = 1 << 8, // __tsan_mutex_not_static // The following flags are runtime private. // Mutex API misuse was detected, so don't report any more. @@ -43,7 +44,8 @@ enum MutexFlags { // Must list all mutex creation flags. MutexCreationFlagMask = MutexFlagLinkerInit | MutexFlagWriteReentrant | - MutexFlagReadReentrant, + MutexFlagReadReentrant | + MutexFlagNotStatic, }; struct SyncVar { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h index 96a18ac4101a..9aef375cbfe6 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h @@ -41,6 +41,8 @@ enum EventType { // u64 addr : 61; // Associated pc. typedef u64 Event; +const uptr kEventPCBits = 61; + struct TraceHeader { #if !SANITIZER_GO BufferedStackTrace stack0; // Start stack for the trace. diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc index 0a87e6e8e3a4..73c69363215a 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc @@ -29,6 +29,7 @@ UBSAN_CHECK(UnsignedIntegerOverflow, "unsigned-integer-overflow", UBSAN_CHECK(IntegerDivideByZero, "integer-divide-by-zero", "integer-divide-by-zero") UBSAN_CHECK(FloatDivideByZero, "float-divide-by-zero", "float-divide-by-zero") +UBSAN_CHECK(InvalidBuiltin, "invalid-builtin-use", "invalid-builtin-use") UBSAN_CHECK(InvalidShiftBase, "invalid-shift-base", "shift-base") UBSAN_CHECK(InvalidShiftExponent, "invalid-shift-exponent", "shift-exponent") UBSAN_CHECK(OutOfBoundsIndex, "out-of-bounds-index", "bounds") diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc index 742802b8f341..d5b68407bb37 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc @@ -26,21 +26,24 @@ using namespace __ubsan; +void __ubsan::GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, + uptr bp, void *context, bool fast) { + uptr top = 0; + uptr bottom = 0; + if (fast) + GetThreadStackTopAndBottom(false, &top, &bottom); + stack->Unwind(max_depth, pc, bp, context, top, bottom, fast); +} + static void MaybePrintStackTrace(uptr pc, uptr bp) { // We assume that flags are already parsed, as UBSan runtime // will definitely be called when we print the first diagnostics message. if (!flags()->print_stacktrace) return; - uptr top = 0; - uptr bottom = 0; - bool request_fast_unwind = common_flags()->fast_unwind_on_fatal; - if (request_fast_unwind) - __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom); - BufferedStackTrace stack; - stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, - request_fast_unwind); + GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, + common_flags()->fast_unwind_on_fatal); stack.Print(); } @@ -97,9 +100,7 @@ class Decorator : public SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() {} const char *Highlight() const { return Green(); } - const char *EndHighlight() const { return Default(); } const char *Note() const { return Black(); } - const char *EndNote() const { return Default(); } }; } @@ -295,7 +296,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc, Buffer.append("%c", P == Loc ? '^' : Byte); Buffer.append("%c", Byte); } - Buffer.append("%s\n", Decor.EndHighlight()); + Buffer.append("%s\n", Decor.Default()); // Go over the line again, and print names for the ranges. InRange = 0; @@ -335,7 +336,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc, Diag::~Diag() { // All diagnostics should be printed under report mutex. - CommonSanitizerReportMutex.CheckLocked(); + ScopedReport::CheckLocked(); Decorator Decor; InternalScopedString Buffer(1024); @@ -345,12 +346,12 @@ Diag::~Diag() { switch (Level) { case DL_Error: - Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.EndWarning(), + Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.Default(), Decor.Bold()); break; case DL_Note: - Buffer.append("%s note: %s", Decor.Note(), Decor.EndNote()); + Buffer.append("%s note: %s", Decor.Note(), Decor.Default()); break; } @@ -363,17 +364,15 @@ Diag::~Diag() { PrintMemorySnippet(Decor, Loc.getMemoryLocation(), Ranges, NumRanges, Args); } +ScopedReport::Initializer::Initializer() { InitAsStandaloneIfNecessary(); } + ScopedReport::ScopedReport(ReportOptions Opts, Location SummaryLoc, ErrorType Type) - : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) { - InitAsStandaloneIfNecessary(); - CommonSanitizerReportMutex.Lock(); -} + : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) {} ScopedReport::~ScopedReport() { MaybePrintStackTrace(Opts.pc, Opts.bp); MaybeReportErrorSummary(SummaryLoc, Type); - CommonSanitizerReportMutex.Unlock(); if (flags()->halt_on_error) Die(); } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h index 3edb67a03c1f..7370b1b62421 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h @@ -231,10 +231,19 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET); GET_CALLER_PC_BP; \ ReportOptions Opts = {unrecoverable_handler, pc, bp} +void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, + void *context, bool fast); + /// \brief Instantiate this class before printing diagnostics in the error /// report. This class ensures that reports from different threads and from /// different sanitizers won't be mixed. class ScopedReport { + struct Initializer { + Initializer(); + }; + Initializer initializer_; + ScopedErrorReportLock report_lock_; + ReportOptions Opts; Location SummaryLoc; ErrorType Type; @@ -242,6 +251,8 @@ class ScopedReport { public: ScopedReport(ReportOptions Opts, Location SummaryLoc, ErrorType Type); ~ScopedReport(); + + static void CheckLocked() { ScopedErrorReportLock::CheckLocked(); } }; void InitializeSuppressions(); diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc index df8ed5fcdf6d..1f4a5bd4062b 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc @@ -26,9 +26,10 @@ void __sanitizer_print_stack_trace() { if (request_fast_unwind) __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom); - GET_REPORT_OPTIONS(false); + GET_CURRENT_PC_BP_SP; + (void)sp; BufferedStackTrace stack; - stack.Unwind(kStackTraceMax, Opts.pc, Opts.bp, nullptr, top, bottom, + stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, request_fast_unwind); stack.Print(); } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc index 8e1f40885a58..c4fb278c8367 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc @@ -18,6 +18,8 @@ #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" +#include <stdlib.h> + namespace __ubsan { const char *MaybeCallUbsanDefaultOptions() { @@ -45,7 +47,7 @@ void InitializeFlags() { CommonFlags cf; cf.CopyFrom(*common_flags()); cf.print_summary = false; - cf.external_symbolizer_path = GetEnv("UBSAN_SYMBOLIZER_PATH"); + cf.external_symbolizer_path = getenv("UBSAN_SYMBOLIZER_PATH"); OverrideCommonFlags(cf); } @@ -59,7 +61,7 @@ void InitializeFlags() { // Override from user-specified string. parser.ParseString(MaybeCallUbsanDefaultOptions()); // Override from environment variable. - parser.ParseString(GetEnv("UBSAN_OPTIONS")); + parser.ParseString(getenv("UBSAN_OPTIONS")); InitializeCommonFlags(); if (Verbosity()) ReportUnrecognizedFlags(); diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc index 75a4490a1843..1112ce1cc5ff 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc @@ -437,6 +437,30 @@ void __ubsan::__ubsan_handle_load_invalid_value_abort(InvalidValueData *Data, Die(); } +static void handleInvalidBuiltin(InvalidBuiltinData *Data, ReportOptions Opts) { + SourceLocation Loc = Data->Loc.acquire(); + ErrorType ET = ErrorType::InvalidBuiltin; + + if (ignoreReport(Loc, Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); + + Diag(Loc, DL_Error, + "passing zero to %0, which is not a valid argument") + << ((Data->Kind == BCK_CTZPassedZero) ? "ctz()" : "clz()"); +} + +void __ubsan::__ubsan_handle_invalid_builtin(InvalidBuiltinData *Data) { + GET_REPORT_OPTIONS(true); + handleInvalidBuiltin(Data, Opts); +} +void __ubsan::__ubsan_handle_invalid_builtin_abort(InvalidBuiltinData *Data) { + GET_REPORT_OPTIONS(true); + handleInvalidBuiltin(Data, Opts); + Die(); +} + static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, ValueHandle Function, ReportOptions Opts) { @@ -628,16 +652,32 @@ static void handleCFIBadIcall(CFICheckFailData *Data, ValueHandle Function, } namespace __ubsan { + #ifdef UBSAN_CAN_USE_CXXABI + +#ifdef _WIN32 + +extern "C" void __ubsan_handle_cfi_bad_type_default(CFICheckFailData *Data, + ValueHandle Vtable, + bool ValidVtable, + ReportOptions Opts) { + Die(); +} + +WIN_WEAK_ALIAS(__ubsan_handle_cfi_bad_type, __ubsan_handle_cfi_bad_type_default) +#else SANITIZER_WEAK_ATTRIBUTE -void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable, - bool ValidVtable, ReportOptions Opts); +#endif +void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, + bool ValidVtable, ReportOptions Opts); + #else -static void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable, - bool ValidVtable, ReportOptions Opts) { +void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, + bool ValidVtable, ReportOptions Opts) { Die(); } #endif + } // namespace __ubsan void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data, @@ -647,7 +687,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data, if (Data->CheckKind == CFITCK_ICall) handleCFIBadIcall(Data, Value, Opts); else - HandleCFIBadType(Data, Value, ValidVtable, Opts); + __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts); } void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data, @@ -657,7 +697,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data, if (Data->CheckKind == CFITCK_ICall) handleCFIBadIcall(Data, Value, Opts); else - HandleCFIBadType(Data, Value, ValidVtable, Opts); + __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts); Die(); } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h index 796321b81889..311776b9f22c 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h @@ -122,6 +122,21 @@ struct InvalidValueData { /// \brief Handle a load of an invalid value for the type. RECOVERABLE(load_invalid_value, InvalidValueData *Data, ValueHandle Val) +/// Known builtin check kinds. +/// Keep in sync with the enum of the same name in CodeGenFunction.h +enum BuiltinCheckKind : unsigned char { + BCK_CTZPassedZero, + BCK_CLZPassedZero, +}; + +struct InvalidBuiltinData { + SourceLocation Loc; + unsigned char Kind; +}; + +/// Handle a builtin called in an invalid way. +RECOVERABLE(invalid_builtin, InvalidBuiltinData *Data) + struct FunctionTypeMismatchData { SourceLocation Loc; const TypeDescriptor &Type; @@ -177,6 +192,13 @@ struct CFICheckFailData { /// \brief Handle control flow integrity failures. RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function, uptr VtableIsValid) + +struct ReportOptions; + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_cfi_bad_type( + CFICheckFailData *Data, ValueHandle Vtable, bool ValidVtable, + ReportOptions Opts); + } #endif // UBSAN_HANDLERS_H diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc index d97ec4813ccd..e15abc64ecca 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc @@ -95,8 +95,8 @@ void __ubsan::__ubsan_handle_dynamic_type_cache_miss_abort( } namespace __ubsan { -void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable, - bool ValidVtable, ReportOptions Opts) { +void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, + bool ValidVtable, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); ErrorType ET = ErrorType::CFIBadType; diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc index 307bca37680e..32fc434ad2bc 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc @@ -27,11 +27,7 @@ const char *__ubsan::GetSanititizerToolName() { return "UndefinedBehaviorSanitizer"; } -static enum { - UBSAN_MODE_UNKNOWN = 0, - UBSAN_MODE_STANDALONE, - UBSAN_MODE_PLUGIN -} ubsan_mode; +static bool ubsan_initialized; static StaticSpinMutex ubsan_init_mu; static void CommonInit() { @@ -40,44 +36,30 @@ static void CommonInit() { static void CommonStandaloneInit() { SanitizerToolName = GetSanititizerToolName(); - InitializeFlags(); CacheBinaryName(); + InitializeFlags(); __sanitizer_set_report_path(common_flags()->log_path); AndroidLogInit(); InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); CommonInit(); - ubsan_mode = UBSAN_MODE_STANDALONE; } void __ubsan::InitAsStandalone() { - if (SANITIZER_CAN_USE_PREINIT_ARRAY) { - CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode); - CommonStandaloneInit(); - return; - } SpinMutexLock l(&ubsan_init_mu); - CHECK_NE(UBSAN_MODE_PLUGIN, ubsan_mode); - if (ubsan_mode == UBSAN_MODE_UNKNOWN) + if (!ubsan_initialized) { CommonStandaloneInit(); -} - -void __ubsan::InitAsStandaloneIfNecessary() { - if (SANITIZER_CAN_USE_PREINIT_ARRAY) { - CHECK_NE(UBSAN_MODE_UNKNOWN, ubsan_mode); - return; + ubsan_initialized = true; } - SpinMutexLock l(&ubsan_init_mu); - if (ubsan_mode == UBSAN_MODE_UNKNOWN) - CommonStandaloneInit(); } +void __ubsan::InitAsStandaloneIfNecessary() { return InitAsStandalone(); } + void __ubsan::InitAsPlugin() { -#if !SANITIZER_CAN_USE_PREINIT_ARRAY SpinMutexLock l(&ubsan_init_mu); -#endif - CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode); - CommonInit(); - ubsan_mode = UBSAN_MODE_PLUGIN; + if (!ubsan_initialized) { + CommonInit(); + ubsan_initialized = true; + } } #endif // CAN_SANITIZE_UB diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc index ff1a20efea3d..8bd500025cbc 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc @@ -18,18 +18,17 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "ubsan_init.h" +#include "ubsan_signals_standalone.h" + +namespace __ubsan { -#if SANITIZER_CAN_USE_PREINIT_ARRAY -__attribute__((section(".preinit_array"), used)) -void (*__local_ubsan_preinit)(void) = __ubsan::InitAsStandalone; -#else -// Use a dynamic initializer. class UbsanStandaloneInitializer { public: UbsanStandaloneInitializer() { - __ubsan::InitAsStandalone(); + InitAsStandalone(); + InitializeDeadlySignals(); } }; static UbsanStandaloneInitializer ubsan_standalone_initializer; -#endif // SANITIZER_CAN_USE_PREINIT_ARRAY +} // namespace __ubsan diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc index a69ca57cd7af..9beb3e2ff953 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc @@ -11,14 +11,19 @@ INTERFACE_FUNCTION(__ubsan_handle_add_overflow) INTERFACE_FUNCTION(__ubsan_handle_add_overflow_abort) INTERFACE_FUNCTION(__ubsan_handle_builtin_unreachable) +INTERFACE_FUNCTION(__ubsan_handle_cfi_bad_type) INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail) INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail_abort) INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow) INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow_abort) +INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss) +INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort) INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch) INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort) +INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin) +INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin_abort) INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value) INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value_abort) INTERFACE_FUNCTION(__ubsan_handle_missing_return) diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_platform.h b/contrib/compiler-rt/lib/ubsan/ubsan_platform.h index 1a3bfd6afb81..72eb4199960e 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_platform.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_platform.h @@ -14,12 +14,13 @@ #define UBSAN_PLATFORM_H // Other platforms should be easy to add, and probably work as-is. -#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)) && \ - (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \ - defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) || \ - defined(__s390__)) +#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \ + defined(__NetBSD__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \ + defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) || \ + defined(__s390__)) || (defined(__sun__) && defined(__svr4__)) # define CAN_SANITIZE_UB 1 -#elif defined(_WIN32) +#elif defined(_WIN32) || defined(__Fuchsia__) # define CAN_SANITIZE_UB 1 #else # define CAN_SANITIZE_UB 0 diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc new file mode 100644 index 000000000000..4c6646dd8b0b --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc @@ -0,0 +1,53 @@ +//=-- ubsan_signals_standalone.cc +//------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Installs signal handlers and related interceptors for UBSan standalone. +// +//===----------------------------------------------------------------------===// + +#include "ubsan_platform.h" +#if CAN_SANITIZE_UB +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "ubsan_diag.h" +#include "ubsan_init.h" + +#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +namespace __ubsan { + +#if SANITIZER_FUCHSIA +void InitializeDeadlySignals() {} +#else +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +static void UBsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr); +} + +static bool is_initialized = false; + +void InitializeDeadlySignals() { + if (is_initialized) + return; + is_initialized = true; + InitializeSignalInterceptors(); + InstallDeadlySignalHandlers(&UBsanOnDeadlySignal); +} +#endif + +} // namespace __ubsan + +#endif // CAN_SANITIZE_UB diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h new file mode 100644 index 000000000000..b29c29482ec8 --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h @@ -0,0 +1,25 @@ +//=-- ubsan_signals_standalone.h +//------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Installs signal handlers and related interceptors for UBSan standalone. +// +//===----------------------------------------------------------------------===// + +#ifndef UBSAN_SIGNALS_STANDALONE_H +#define UBSAN_SIGNALS_STANDALONE_H + +namespace __ubsan { + +// Initializes signal handlers and interceptors. +void InitializeDeadlySignals(); + +} // namespace __ubsan + +#endif // UBSAN_SIGNALS_STANDALONE_H diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra b/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra new file mode 100644 index 000000000000..7f8be694401a --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra @@ -0,0 +1 @@ +__ubsan_* diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc new file mode 100644 index 000000000000..5a5675c983fe --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc @@ -0,0 +1,104 @@ +#include "sanitizer_common/sanitizer_atomic.h" + +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> + +#ifdef KERNEL_USE +extern "C" void ubsan_message(const char *msg); +static void message(const char *msg) { ubsan_message(msg); } +#else +static void message(const char *msg) { + write(2, msg, strlen(msg)); +} +#endif + +static const int kMaxCallerPcs = 20; +static __sanitizer::atomic_uintptr_t caller_pcs[kMaxCallerPcs]; +// Number of elements in caller_pcs. A special value of kMaxCallerPcs + 1 means +// that "too many errors" has already been reported. +static __sanitizer::atomic_uint32_t caller_pcs_sz; + +__attribute__((noinline)) static bool report_this_error(void *caller_p) { + uintptr_t caller = reinterpret_cast<uintptr_t>(caller_p); + if (caller == 0) return false; + while (true) { + unsigned sz = __sanitizer::atomic_load_relaxed(&caller_pcs_sz); + if (sz > kMaxCallerPcs) return false; // early exit + // when sz==kMaxCallerPcs print "too many errors", but only when cmpxchg + // succeeds in order to not print it multiple times. + if (sz > 0 && sz < kMaxCallerPcs) { + uintptr_t p; + for (unsigned i = 0; i < sz; ++i) { + p = __sanitizer::atomic_load_relaxed(&caller_pcs[i]); + if (p == 0) break; // Concurrent update. + if (p == caller) return false; + } + if (p == 0) continue; // FIXME: yield? + } + + if (!__sanitizer::atomic_compare_exchange_strong( + &caller_pcs_sz, &sz, sz + 1, __sanitizer::memory_order_seq_cst)) + continue; // Concurrent update! Try again from the start. + + if (sz == kMaxCallerPcs) { + message("ubsan: too many errors\n"); + return false; + } + __sanitizer::atomic_store_relaxed(&caller_pcs[sz], caller); + return true; + } +} + +#if defined(__ANDROID__) +extern "C" __attribute__((weak)) void android_set_abort_message(const char *); +static void abort_with_message(const char *msg) { + if (&android_set_abort_message) android_set_abort_message(msg); + abort(); +} +#else +static void abort_with_message(const char *) { abort(); } +#endif + +#define INTERFACE extern "C" __attribute__((visibility("default"))) + +// FIXME: add caller pc to the error message (possibly as "ubsan: error-type +// @1234ABCD"). +#define HANDLER_RECOVER(name, msg) \ + INTERFACE void __ubsan_handle_##name##_minimal() { \ + if (!report_this_error(__builtin_return_address(0))) return; \ + message("ubsan: " msg "\n"); \ + } + +#define HANDLER_NORECOVER(name, msg) \ + INTERFACE void __ubsan_handle_##name##_minimal_abort() { \ + message("ubsan: " msg "\n"); \ + abort_with_message("ubsan: " msg); \ + } + +#define HANDLER(name, msg) \ + HANDLER_RECOVER(name, msg) \ + HANDLER_NORECOVER(name, msg) + +HANDLER(type_mismatch, "type-mismatch") +HANDLER(add_overflow, "add-overflow") +HANDLER(sub_overflow, "sub-overflow") +HANDLER(mul_overflow, "mul-overflow") +HANDLER(negate_overflow, "negate-overflow") +HANDLER(divrem_overflow, "divrem-overflow") +HANDLER(shift_out_of_bounds, "shift-out-of-bounds") +HANDLER(out_of_bounds, "out-of-bounds") +HANDLER_RECOVER(builtin_unreachable, "builtin-unreachable") +HANDLER_RECOVER(missing_return, "missing-return") +HANDLER(vla_bound_not_positive, "vla-bound-not-positive") +HANDLER(float_cast_overflow, "float-cast-overflow") +HANDLER(load_invalid_value, "load-invalid-value") +HANDLER(invalid_builtin, "invalid-builtin") +HANDLER(function_type_mismatch, "function-type-mismatch") +HANDLER(nonnull_arg, "nonnull-arg") +HANDLER(nonnull_return, "nonnull-return") +HANDLER(nullability_arg, "nullability-arg") +HANDLER(nullability_return, "nullability-return") +HANDLER(pointer_overflow, "pointer-overflow") +HANDLER(cfi_check_fail, "cfi-check-fail") diff --git a/contrib/compiler-rt/lib/xray/weak_symbols.txt b/contrib/compiler-rt/lib/xray/weak_symbols.txt new file mode 100644 index 000000000000..963fff2d697e --- /dev/null +++ b/contrib/compiler-rt/lib/xray/weak_symbols.txt @@ -0,0 +1,4 @@ +___start_xray_fn_idx +___start_xray_instr_map +___stop_xray_fn_idx +___stop_xray_instr_map diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc index 7ba755ac3069..a0018f6b0cba 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc @@ -13,28 +13,34 @@ // //===----------------------------------------------------------------------===// #include "xray_buffer_queue.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" -#include <cstdlib> -#include <tuple> - using namespace __xray; using namespace __sanitizer; -BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success) - : BufferSize(B), Buffers(N), Mutex(), OwnedBuffers(), Finalizing{0} { - for (auto &T : Buffers) { - void *Tmp = malloc(BufferSize); +BufferQueue::BufferQueue(size_t B, size_t N, bool &Success) + : BufferSize(B), Buffers(new BufferRep[N]()), BufferCount(N), Finalizing{0}, + OwnedBuffers(new void *[N]()), Next(Buffers), First(Buffers), + LiveBuffers(0) { + for (size_t i = 0; i < N; ++i) { + auto &T = Buffers[i]; + void *Tmp = InternalAlloc(BufferSize, nullptr, 64); if (Tmp == nullptr) { Success = false; return; } - - auto &Buf = std::get<0>(T); + void *Extents = InternalAlloc(sizeof(BufferExtents), nullptr, 64); + if (Extents == nullptr) { + Success = false; + return; + } + auto &Buf = T.Buff; Buf.Buffer = Tmp; Buf.Size = B; - OwnedBuffers.emplace(Tmp); + Buf.Extents = reinterpret_cast<BufferExtents *>(Extents); + OwnedBuffers[i] = Tmp; } Success = true; } @@ -42,27 +48,50 @@ BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success) BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) { if (__sanitizer::atomic_load(&Finalizing, __sanitizer::memory_order_acquire)) return ErrorCode::QueueFinalizing; - __sanitizer::BlockingMutexLock Guard(&Mutex); - if (Buffers.empty()) + __sanitizer::SpinMutexLock Guard(&Mutex); + if (LiveBuffers == BufferCount) return ErrorCode::NotEnoughMemory; - auto &T = Buffers.front(); - auto &B = std::get<0>(T); + + auto &T = *Next; + auto &B = T.Buff; Buf = B; - B.Buffer = nullptr; - B.Size = 0; - Buffers.pop_front(); + T.Used = true; + ++LiveBuffers; + + if (++Next == (Buffers + BufferCount)) + Next = Buffers; + return ErrorCode::Ok; } BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) { - if (OwnedBuffers.count(Buf.Buffer) == 0) + // Blitz through the buffers array to find the buffer. + bool Found = false; + for (auto I = OwnedBuffers, E = OwnedBuffers + BufferCount; I != E; ++I) { + if (*I == Buf.Buffer) { + Found = true; + break; + } + } + if (!Found) return ErrorCode::UnrecognizedBuffer; - __sanitizer::BlockingMutexLock Guard(&Mutex); + + __sanitizer::SpinMutexLock Guard(&Mutex); + + // This points to a semantic bug, we really ought to not be releasing more + // buffers than we actually get. + if (LiveBuffers == 0) + return ErrorCode::NotEnoughMemory; // Now that the buffer has been released, we mark it as "used". - Buffers.emplace(Buffers.end(), Buf, true /* used */); + First->Buff = Buf; + First->Used = true; Buf.Buffer = nullptr; Buf.Size = 0; + --LiveBuffers; + if (++First == (Buffers + BufferCount)) + First = Buffers; + return ErrorCode::Ok; } @@ -74,8 +103,12 @@ BufferQueue::ErrorCode BufferQueue::finalize() { } BufferQueue::~BufferQueue() { - for (auto &T : Buffers) { - auto &Buf = std::get<0>(T); - free(Buf.Buffer); + for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) { + auto &T = *I; + auto &Buf = T.Buff; + InternalFree(Buf.Buffer); + InternalFree(Buf.Extents); } + delete[] Buffers; + delete[] OwnedBuffers; } diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h index e051695a297b..1ceb58274616 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h @@ -15,11 +15,9 @@ #ifndef XRAY_BUFFER_QUEUE_H #define XRAY_BUFFER_QUEUE_H +#include <cstddef> #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_mutex.h" -#include <deque> -#include <unordered_set> -#include <utility> namespace __xray { @@ -29,23 +27,50 @@ namespace __xray { /// the "flight data recorder" (FDR) mode to support ongoing XRay function call /// trace collection. class BufferQueue { -public: + public: + struct alignas(64) BufferExtents { + __sanitizer::atomic_uint64_t Size; + }; + struct Buffer { void *Buffer = nullptr; size_t Size = 0; + BufferExtents* Extents; + }; + + private: + struct BufferRep { + // The managed buffer. + Buffer Buff; + + // This is true if the buffer has been returned to the available queue, and + // is considered "used" by another thread. + bool Used = false; }; -private: + // Size of each individual Buffer. size_t BufferSize; - // We use a bool to indicate whether the Buffer has been used in this - // freelist implementation. - std::deque<std::tuple<Buffer, bool>> Buffers; - __sanitizer::BlockingMutex Mutex; - std::unordered_set<void *> OwnedBuffers; + BufferRep *Buffers; + size_t BufferCount; + + __sanitizer::SpinMutex Mutex; __sanitizer::atomic_uint8_t Finalizing; -public: + // Pointers to buffers managed/owned by the BufferQueue. + void **OwnedBuffers; + + // Pointer to the next buffer to be handed out. + BufferRep *Next; + + // Pointer to the entry in the array where the next released buffer will be + // placed. + BufferRep *First; + + // Count of buffers that have been handed out through 'getBuffer'. + size_t LiveBuffers; + + public: enum class ErrorCode : unsigned { Ok, NotEnoughMemory, @@ -56,16 +81,16 @@ public: static const char *getErrorString(ErrorCode E) { switch (E) { - case ErrorCode::Ok: - return "(none)"; - case ErrorCode::NotEnoughMemory: - return "no available buffers in the queue"; - case ErrorCode::QueueFinalizing: - return "queue already finalizing"; - case ErrorCode::UnrecognizedBuffer: - return "buffer being returned not owned by buffer queue"; - case ErrorCode::AlreadyFinalized: - return "queue already finalized"; + case ErrorCode::Ok: + return "(none)"; + case ErrorCode::NotEnoughMemory: + return "no available buffers in the queue"; + case ErrorCode::QueueFinalizing: + return "queue already finalizing"; + case ErrorCode::UnrecognizedBuffer: + return "buffer being returned not owned by buffer queue"; + case ErrorCode::AlreadyFinalized: + return "queue already finalized"; } return "unknown error"; } @@ -82,15 +107,18 @@ public: /// - BufferQueue is not finalising. /// /// Returns: - /// - std::errc::not_enough_memory on exceeding MaxSize. - /// - no error when we find a Buffer. - /// - std::errc::state_not_recoverable on finalising BufferQueue. + /// - ErrorCode::NotEnoughMemory on exceeding MaxSize. + /// - ErrorCode::Ok when we find a Buffer. + /// - ErrorCode::QueueFinalizing or ErrorCode::AlreadyFinalized on + /// a finalizing/finalized BufferQueue. ErrorCode getBuffer(Buffer &Buf); /// Updates |Buf| to point to nullptr, with size 0. /// /// Returns: - /// - ... + /// - ErrorCode::Ok when we successfully release the buffer. + /// - ErrorCode::UnrecognizedBuffer for when this BufferQueue does not own + /// the buffer being released. ErrorCode releaseBuffer(Buffer &Buf); bool finalizing() const { @@ -107,17 +135,18 @@ public: /// - All releaseBuffer operations will not fail. /// /// After a call to finalize succeeds, all subsequent calls to finalize will - /// fail with std::errc::state_not_recoverable. + /// fail with ErrorCode::QueueFinalizing. ErrorCode finalize(); /// Applies the provided function F to each Buffer in the queue, only if the /// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a - /// releaseBuffer(...) operation. - template <class F> void apply(F Fn) { - __sanitizer::BlockingMutexLock G(&Mutex); - for (const auto &T : Buffers) { - if (std::get<1>(T)) - Fn(std::get<0>(T)); + /// releaseBuffer(...) operation). + template <class F> + void apply(F Fn) { + __sanitizer::SpinMutexLock G(&Mutex); + for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) { + const auto &T = *I; + if (T.Used) Fn(T.Buff); } } @@ -125,6 +154,6 @@ public: ~BufferQueue(); }; -} // namespace __xray +} // namespace __xray -#endif // XRAY_BUFFER_QUEUE_H +#endif // XRAY_BUFFER_QUEUE_H diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h index 3d6d38892c76..324208db82ca 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h @@ -30,7 +30,10 @@ struct alignas(16) MetadataRecord { TSCWrap, WalltimeMarker, CustomEventMarker, + CallArgument, + BufferExtents, }; + // Use 7 bits to identify this record type. /* RecordKinds */ uint8_t RecordKind : 7; char Data[15]; diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc index a7e1382c3865..1bfa10c21f5c 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc @@ -15,15 +15,11 @@ // //===----------------------------------------------------------------------===// #include "xray_fdr_logging.h" -#include <algorithm> -#include <bitset> -#include <cerrno> -#include <cstring> +#include <errno.h> #include <sys/syscall.h> #include <sys/time.h> #include <time.h> #include <unistd.h> -#include <unordered_map> #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" @@ -39,7 +35,7 @@ namespace __xray { // Global BufferQueue. -std::shared_ptr<BufferQueue> BQ; +BufferQueue *BQ = nullptr; __sanitizer::atomic_sint32_t LogFlushStatus = { XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; @@ -52,19 +48,31 @@ __sanitizer::SpinMutex FDROptionsMutex; XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { if (__sanitizer::atomic_load(&LoggingStatus, __sanitizer::memory_order_acquire) != - XRayLogInitStatus::XRAY_LOG_FINALIZED) + XRayLogInitStatus::XRAY_LOG_FINALIZED) { + if (__sanitizer::Verbosity()) + Report("Not flushing log, implementation is not finalized.\n"); return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + } s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; if (!__sanitizer::atomic_compare_exchange_strong( &LogFlushStatus, &Result, XRayLogFlushStatus::XRAY_LOG_FLUSHING, - __sanitizer::memory_order_release)) + __sanitizer::memory_order_release)) { + + if (__sanitizer::Verbosity()) + Report("Not flushing log, implementation is still finalizing.\n"); return static_cast<XRayLogFlushStatus>(Result); + } - // Make a copy of the BufferQueue pointer to prevent other threads that may be - // resetting it from blowing away the queue prematurely while we're dealing - // with it. - auto LocalBQ = BQ; + if (BQ == nullptr) { + if (__sanitizer::Verbosity()) + Report("Cannot flush when global buffer queue is null.\n"); + return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + } + + // We wait a number of milliseconds to allow threads to see that we've + // finalised before attempting to flush the log. + __sanitizer::SleepForMillis(flags()->xray_fdr_log_grace_period_ms); // We write out the file in the following format: // @@ -95,24 +103,44 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { TSCSupported ? getTSCFrequency() : __xray::NanosecondsPerSecond; XRayFileHeader Header; - Header.Version = 1; + + // Version 2 of the log writes the extents of the buffer, instead of relying + // on an end-of-buffer record. + Header.Version = 2; Header.Type = FileTypes::FDR_LOG; Header.CycleFrequency = CycleFrequency; + // FIXME: Actually check whether we have 'constant_tsc' and 'nonstop_tsc' // before setting the values in the header. Header.ConstantTSC = 1; Header.NonstopTSC = 1; - Header.FdrData = FdrAdditionalHeaderData{LocalBQ->ConfiguredBufferSize()}; + Header.FdrData = FdrAdditionalHeaderData{BQ->ConfiguredBufferSize()}; retryingWriteAll(Fd, reinterpret_cast<char *>(&Header), reinterpret_cast<char *>(&Header) + sizeof(Header)); - LocalBQ->apply([&](const BufferQueue::Buffer &B) { - uint64_t BufferSize = B.Size; - if (BufferSize > 0) { + BQ->apply([&](const BufferQueue::Buffer &B) { + // Starting at version 2 of the FDR logging implementation, we only write + // the records identified by the extents of the buffer. We use the Extents + // from the Buffer and write that out as the first record in the buffer. + // We still use a Metadata record, but fill in the extents instead for the + // data. + MetadataRecord ExtentsRecord; + auto BufferExtents = __sanitizer::atomic_load( + &B.Extents->Size, __sanitizer::memory_order_acquire); + assert(BufferExtents <= B.Size); + ExtentsRecord.Type = uint8_t(RecordType::Metadata); + ExtentsRecord.RecordKind = + uint8_t(MetadataRecord::RecordKinds::BufferExtents); + std::memcpy(ExtentsRecord.Data, &BufferExtents, sizeof(BufferExtents)); + if (BufferExtents > 0) { + retryingWriteAll(Fd, reinterpret_cast<char *>(&ExtentsRecord), + reinterpret_cast<char *>(&ExtentsRecord) + + sizeof(MetadataRecord)); retryingWriteAll(Fd, reinterpret_cast<char *>(B.Buffer), - reinterpret_cast<char *>(B.Buffer) + B.Size); + reinterpret_cast<char *>(B.Buffer) + BufferExtents); } }); + __sanitizer::atomic_store(&LogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED, __sanitizer::memory_order_release); @@ -124,8 +152,11 @@ XRayLogInitStatus fdrLoggingFinalize() XRAY_NEVER_INSTRUMENT { if (!__sanitizer::atomic_compare_exchange_strong( &LoggingStatus, &CurrentStatus, XRayLogInitStatus::XRAY_LOG_FINALIZING, - __sanitizer::memory_order_release)) + __sanitizer::memory_order_release)) { + if (__sanitizer::Verbosity()) + Report("Cannot finalize log, implementation not initialized.\n"); return static_cast<XRayLogInitStatus>(CurrentStatus); + } // Do special things to make the log finalize itself, and not allow any more // operations to be performed until re-initialized. @@ -146,7 +177,8 @@ XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT { return static_cast<XRayLogInitStatus>(CurrentStatus); // Release the in-memory buffer queue. - BQ.reset(); + delete BQ; + BQ = nullptr; // Spin until the flushing status is flushed. s32 CurrentFlushingStatus = XRayLogFlushStatus::XRAY_LOG_FLUSHED; @@ -163,19 +195,22 @@ XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT { return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } -static std::tuple<uint64_t, unsigned char> -getTimestamp() XRAY_NEVER_INSTRUMENT { +struct TSCAndCPU { + uint64_t TSC = 0; + unsigned char CPU = 0; +}; + +static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT { // We want to get the TSC as early as possible, so that we can check whether // we've seen this CPU before. We also do it before we load anything else, to // allow for forward progress with the scheduling. - unsigned char CPU; - uint64_t TSC; + TSCAndCPU Result; // Test once for required CPU features static bool TSCSupported = probeRequiredCPUFeatures(); if (TSCSupported) { - TSC = __xray::readTSC(CPU); + Result.TSC = __xray::readTSC(Result.CPU); } else { // FIXME: This code needs refactoring as it appears in multiple locations timespec TS; @@ -184,32 +219,35 @@ getTimestamp() XRAY_NEVER_INSTRUMENT { Report("clock_gettime(2) return %d, errno=%d", result, int(errno)); TS = {0, 0}; } - CPU = 0; - TSC = TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec; + Result.CPU = 0; + Result.TSC = TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec; } - return std::make_tuple(TSC, CPU); + return Result; } void fdrLoggingHandleArg0(int32_t FuncId, XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { - auto TSC_CPU = getTimestamp(); - __xray_fdr_internal::processFunctionHook(FuncId, Entry, std::get<0>(TSC_CPU), - std::get<1>(TSC_CPU), clock_gettime, - LoggingStatus, BQ); + auto TC = getTimestamp(); + __xray_fdr_internal::processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, 0, + clock_gettime, BQ); +} + +void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry, + uint64_t Arg) XRAY_NEVER_INSTRUMENT { + auto TC = getTimestamp(); + __xray_fdr_internal::processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, Arg, + clock_gettime, BQ); } void fdrLoggingHandleCustomEvent(void *Event, std::size_t EventSize) XRAY_NEVER_INSTRUMENT { using namespace __xray_fdr_internal; - auto TSC_CPU = getTimestamp(); - auto &TSC = std::get<0>(TSC_CPU); - auto &CPU = std::get<1>(TSC_CPU); - thread_local bool Running = false; + auto TC = getTimestamp(); + auto &TSC = TC.TSC; + auto &CPU = TC.CPU; RecursionGuard Guard{Running}; - if (!Guard) { - assert(Running && "RecursionGuard is buggy!"); + if (!Guard) return; - } if (EventSize > std::numeric_limits<int32_t>::max()) { using Empty = struct {}; static Empty Once = [&] { @@ -220,15 +258,16 @@ void fdrLoggingHandleCustomEvent(void *Event, (void)Once; } int32_t ReducedEventSize = static_cast<int32_t>(EventSize); - if (!isLogInitializedAndReady(LocalBQ, TSC, CPU, clock_gettime)) + auto &TLD = getThreadLocalData(); + if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, clock_gettime)) return; // Here we need to prepare the log to handle: // - The metadata record we're going to write. (16 bytes) // - The additional data we're going to write. Currently, that's the size of // the event we're going to dump into the log as free-form bytes. - if (!prepareBuffer(clock_gettime, MetadataRecSize + EventSize)) { - LocalBQ = nullptr; + if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) { + TLD.BQ = nullptr; return; } @@ -240,27 +279,35 @@ void fdrLoggingHandleCustomEvent(void *Event, CustomEvent.Type = uint8_t(RecordType::Metadata); CustomEvent.RecordKind = uint8_t(MetadataRecord::RecordKinds::CustomEventMarker); - constexpr auto TSCSize = sizeof(std::get<0>(TSC_CPU)); + constexpr auto TSCSize = sizeof(TC.TSC); std::memcpy(&CustomEvent.Data, &ReducedEventSize, sizeof(int32_t)); std::memcpy(&CustomEvent.Data[sizeof(int32_t)], &TSC, TSCSize); - std::memcpy(RecordPtr, &CustomEvent, sizeof(CustomEvent)); - RecordPtr += sizeof(CustomEvent); - std::memcpy(RecordPtr, Event, ReducedEventSize); + std::memcpy(TLD.RecordPtr, &CustomEvent, sizeof(CustomEvent)); + TLD.RecordPtr += sizeof(CustomEvent); + std::memcpy(TLD.RecordPtr, Event, ReducedEventSize); + incrementExtents(MetadataRecSize + EventSize); endBufferIfFull(); } XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { - if (OptionsSize != sizeof(FDRLoggingOptions)) + if (OptionsSize != sizeof(FDRLoggingOptions)) { + if (__sanitizer::Verbosity()) + Report("Cannot initialize FDR logging; wrong size for options: %d\n", + OptionsSize); return static_cast<XRayLogInitStatus>(__sanitizer::atomic_load( &LoggingStatus, __sanitizer::memory_order_acquire)); + } s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; if (!__sanitizer::atomic_compare_exchange_strong( &LoggingStatus, &CurrentStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZING, - __sanitizer::memory_order_release)) + __sanitizer::memory_order_release)) { + if (__sanitizer::Verbosity()) + Report("Cannot initialize already initialized implementation.\n"); return static_cast<XRayLogInitStatus>(CurrentStatus); + } { __sanitizer::SpinMutexLock Guard(&FDROptionsMutex); @@ -268,12 +315,40 @@ XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax, } bool Success = false; - BQ = std::make_shared<BufferQueue>(BufferSize, BufferMax, Success); + + if (BQ != nullptr) { + delete BQ; + BQ = nullptr; + } + + if (BQ == nullptr) + BQ = new BufferQueue(BufferSize, BufferMax, Success); + if (!Success) { Report("BufferQueue init failed.\n"); + if (BQ != nullptr) { + delete BQ; + BQ = nullptr; + } return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } + static bool UNUSED Once = [] { + pthread_key_create(&__xray_fdr_internal::Key, +[](void *) { + auto &TLD = __xray_fdr_internal::getThreadLocalData(); + if (TLD.BQ == nullptr) + return; + auto EC = TLD.BQ->releaseBuffer(TLD.Buffer); + if (EC != BufferQueue::ErrorCode::Ok) + Report("At thread exit, failed to release buffer at %p; error=%s\n", + TLD.Buffer.Buffer, BufferQueue::getErrorString(EC)); + }); + return false; + }(); + + // Arg1 handler should go in first to avoid concurrent code accidentally + // falling back to arg0 when it should have ran arg1. + __xray_set_handler_arg1(fdrLoggingHandleArg1); // Install the actual handleArg0 handler after initialising the buffers. __xray_set_handler(fdrLoggingHandleArg0); __xray_set_customevent_handler(fdrLoggingHandleCustomEvent); @@ -281,20 +356,31 @@ XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax, __sanitizer::atomic_store(&LoggingStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZED, __sanitizer::memory_order_release); - Report("XRay FDR init successful.\n"); + + if (__sanitizer::Verbosity()) + Report("XRay FDR init successful.\n"); return XRayLogInitStatus::XRAY_LOG_INITIALIZED; } -} // namespace __xray - -static auto UNUSED Unused = [] { +bool fdrLogDynamicInitializer() XRAY_NEVER_INSTRUMENT { using namespace __xray; - if (flags()->xray_fdr_log) { - XRayLogImpl Impl{ - fdrLoggingInit, fdrLoggingFinalize, fdrLoggingHandleArg0, - fdrLoggingFlush, - }; + XRayLogImpl Impl{ + fdrLoggingInit, + fdrLoggingFinalize, + fdrLoggingHandleArg0, + fdrLoggingFlush, + }; + auto RegistrationResult = __xray_log_register_mode("xray-fdr", Impl); + if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && + __sanitizer::Verbosity()) + Report("Cannot register XRay FDR mode to 'xray-fdr'; error = %d\n", + RegistrationResult); + if (flags()->xray_fdr_log || + !__sanitizer::internal_strcmp(flags()->xray_mode, "xray-fdr")) __xray_set_log_impl(Impl); - } return true; -}(); +} + +} // namespace __xray + +static auto UNUSED Unused = __xray::fdrLogDynamicInitializer(); diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.h b/contrib/compiler-rt/lib/xray/xray_fdr_logging.h index 426b54dc7884..1639d550a44c 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.h @@ -30,6 +30,7 @@ XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax, void *Options, size_t OptionsSize); XRayLogInitStatus fdrLoggingFinalize(); void fdrLoggingHandleArg0(int32_t FuncId, XRayEntryType Entry); +void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry, uint64_t Arg1); XRayLogFlushStatus fdrLoggingFlush(); XRayLogInitStatus fdrLoggingReset(); diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h b/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h index 4a1d80fd0eba..59eab55b2573 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h @@ -18,13 +18,13 @@ #define XRAY_XRAY_FDR_LOGGING_IMPL_H #include <cassert> -#include <cstdint> +#include <cstddef> #include <cstring> #include <limits> -#include <memory> -#include <string> +#include <pthread.h> #include <sys/syscall.h> #include <time.h> +#include <type_traits> #include <unistd.h> #include "sanitizer_common/sanitizer_common.h" @@ -52,132 +52,120 @@ __sanitizer::atomic_sint32_t LoggingStatus = { /// cooperation with xray_fdr_logging class, so be careful and think twice. namespace __xray_fdr_internal { -/// Writes the new buffer record and wallclock time that begin a buffer for a -/// thread to MemPtr and increments MemPtr. Bypasses the thread local state -/// machine and writes directly to memory without checks. -static void writeNewBufferPreamble(pid_t Tid, timespec TS, char *&MemPtr); +/// Writes the new buffer record and wallclock time that begin a buffer for the +/// current thread. +static void writeNewBufferPreamble(pid_t Tid, timespec TS); -/// Write a metadata record to switch to a new CPU to MemPtr and increments -/// MemPtr. Bypasses the thread local state machine and writes directly to -/// memory without checks. -static void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC, char *&MemPtr); - -/// Writes an EOB metadata record to MemPtr and increments MemPtr. Bypasses the -/// thread local state machine and writes directly to memory without checks. -static void writeEOBMetadata(char *&MemPtr); - -/// Writes a TSC Wrap metadata record to MemPtr and increments MemPtr. Bypasses -/// the thread local state machine and directly writes to memory without checks. -static void writeTSCWrapMetadata(uint64_t TSC, char *&MemPtr); - -/// Writes a Function Record to MemPtr and increments MemPtr. Bypasses the -/// thread local state machine and writes the function record directly to -/// memory. +/// Writes a Function Record to the buffer associated with the current thread. static void writeFunctionRecord(int FuncId, uint32_t TSCDelta, - XRayEntryType EntryType, char *&MemPtr); + XRayEntryType EntryType); /// Sets up a new buffer in thread_local storage and writes a preamble. The /// wall_clock_reader function is used to populate the WallTimeRecord entry. static void setupNewBuffer(int (*wall_clock_reader)(clockid_t, struct timespec *)); -/// Called to record CPU time for a new CPU within the current thread. -static void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC); - -/// Called to close the buffer when the thread exhausts the buffer or when the -/// thread exits (via a thread local variable destructor). -static void writeEOBMetadata(); - /// TSC Wrap records are written when a TSC delta encoding scheme overflows. static void writeTSCWrapMetadata(uint64_t TSC); -/// Here's where the meat of the processing happens. The writer captures -/// function entry, exit and tail exit points with a time and will create -/// TSCWrap, NewCPUId and Function records as necessary. The writer might -/// walk backward through its buffer and erase trivial functions to avoid -/// polluting the log and may use the buffer queue to obtain or release a -/// buffer. -static void processFunctionHook(int32_t FuncId, XRayEntryType Entry, - uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, - struct timespec *), - __sanitizer::atomic_sint32_t &LoggingStatus, - const std::shared_ptr<BufferQueue> &BQ); +// Group together thread-local-data in a struct, then hide it behind a function +// call so that it can be initialized on first use instead of as a global. We +// force the alignment to 64-bytes for x86 cache line alignment, as this +// structure is used in the hot path of implementation. +struct alignas(64) ThreadLocalData { + BufferQueue::Buffer Buffer; + char *RecordPtr = nullptr; + // The number of FunctionEntry records immediately preceding RecordPtr. + uint8_t NumConsecutiveFnEnters = 0; -//-----------------------------------------------------------------------------| -// The rest of the file is implementation. | -//-----------------------------------------------------------------------------| -// Functions are implemented in the header for inlining since we don't want | -// to grow the stack when we've hijacked the binary for logging. | -//-----------------------------------------------------------------------------| + // The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit + // records preceding RecordPtr. + uint8_t NumTailCalls = 0; -namespace { + // We use a thread_local variable to keep track of which CPUs we've already + // run, and the TSC times for these CPUs. This allows us to stop repeating the + // CPU field in the function records. + // + // We assume that we'll support only 65536 CPUs for x86_64. + uint16_t CurrentCPU = std::numeric_limits<uint16_t>::max(); + uint64_t LastTSC = 0; + uint64_t LastFunctionEntryTSC = 0; -thread_local BufferQueue::Buffer Buffer; -thread_local char *RecordPtr = nullptr; + // Make sure a thread that's ever called handleArg0 has a thread-local + // live reference to the buffer queue for this particular instance of + // FDRLogging, and that we're going to clean it up when the thread exits. + BufferQueue *BQ = nullptr; +}; -// The number of FunctionEntry records immediately preceding RecordPtr. -thread_local uint8_t NumConsecutiveFnEnters = 0; +static_assert(std::is_trivially_destructible<ThreadLocalData>::value, + "ThreadLocalData must be trivially destructible"); -// The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit -// records preceding RecordPtr. -thread_local uint8_t NumTailCalls = 0; +static constexpr auto MetadataRecSize = sizeof(MetadataRecord); +static constexpr auto FunctionRecSize = sizeof(FunctionRecord); -constexpr auto MetadataRecSize = sizeof(MetadataRecord); -constexpr auto FunctionRecSize = sizeof(FunctionRecord); +// Use a global pthread key to identify thread-local data for logging. +static pthread_key_t Key; -// We use a thread_local variable to keep track of which CPUs we've already -// run, and the TSC times for these CPUs. This allows us to stop repeating the -// CPU field in the function records. +// This function will initialize the thread-local data structure used by the FDR +// logging implementation and return a reference to it. The implementation +// details require a bit of care to maintain. // -// We assume that we'll support only 65536 CPUs for x86_64. -thread_local uint16_t CurrentCPU = std::numeric_limits<uint16_t>::max(); -thread_local uint64_t LastTSC = 0; -thread_local uint64_t LastFunctionEntryTSC = 0; - -class ThreadExitBufferCleanup { - std::shared_ptr<BufferQueue> &Buffers; - BufferQueue::Buffer &Buffer; - -public: - explicit ThreadExitBufferCleanup(std::shared_ptr<BufferQueue> &BQ, - BufferQueue::Buffer &Buffer) - XRAY_NEVER_INSTRUMENT : Buffers(BQ), - Buffer(Buffer) {} - - ~ThreadExitBufferCleanup() noexcept XRAY_NEVER_INSTRUMENT { - if (RecordPtr == nullptr) - return; +// First, some requirements on the implementation in general: +// +// - XRay handlers should not call any memory allocation routines that may +// delegate to an instrumented implementation. This means functions like +// malloc() and free() should not be called while instrumenting. +// +// - We would like to use some thread-local data initialized on first-use of +// the XRay instrumentation. These allow us to implement unsynchronized +// routines that access resources associated with the thread. +// +// The implementation here uses a few mechanisms that allow us to provide both +// the requirements listed above. We do this by: +// +// 1. Using a thread-local aligned storage buffer for representing the +// ThreadLocalData struct. This data will be uninitialized memory by +// design. +// +// 2. Not requiring a thread exit handler/implementation, keeping the +// thread-local as purely a collection of references/data that do not +// require cleanup. +// +// We're doing this to avoid using a `thread_local` object that has a +// non-trivial destructor, because the C++ runtime might call std::malloc(...) +// to register calls to destructors. Deadlocks may arise when, for example, an +// externally provided malloc implementation is XRay instrumented, and +// initializing the thread-locals involves calling into malloc. A malloc +// implementation that does global synchronization might be holding a lock for a +// critical section, calling a function that might be XRay instrumented (and +// thus in turn calling into malloc by virtue of registration of the +// thread_local's destructor). +static ThreadLocalData &getThreadLocalData() { + static_assert(alignof(ThreadLocalData) >= 64, + "ThreadLocalData must be cache line aligned."); + thread_local ThreadLocalData TLD; + thread_local bool UNUSED ThreadOnce = [] { + pthread_setspecific(Key, &TLD); + return false; + }(); + return TLD; +} - // We make sure that upon exit, a thread will write out the EOB - // MetadataRecord in the thread-local log, and also release the buffer to - // the queue. - assert((RecordPtr + MetadataRecSize) - static_cast<char *>(Buffer.Buffer) >= - static_cast<ptrdiff_t>(MetadataRecSize)); - if (Buffers) { - writeEOBMetadata(); - auto EC = Buffers->releaseBuffer(Buffer); - if (EC != BufferQueue::ErrorCode::Ok) - Report("Failed to release buffer at %p; error=%s\n", Buffer.Buffer, - BufferQueue::getErrorString(EC)); - Buffers = nullptr; - return; - } - } -}; +//-----------------------------------------------------------------------------| +// The rest of the file is implementation. | +//-----------------------------------------------------------------------------| +// Functions are implemented in the header for inlining since we don't want | +// to grow the stack when we've hijacked the binary for logging. | +//-----------------------------------------------------------------------------| -// Make sure a thread that's ever called handleArg0 has a thread-local -// live reference to the buffer queue for this particular instance of -// FDRLogging, and that we're going to clean it up when the thread exits. -thread_local std::shared_ptr<BufferQueue> LocalBQ = nullptr; -thread_local ThreadExitBufferCleanup Cleanup(LocalBQ, Buffer); +namespace { class RecursionGuard { - bool &Running; + volatile bool &Running; const bool Valid; public: - explicit RecursionGuard(bool &R) : Running(R), Valid(!R) { + explicit RecursionGuard(volatile bool &R) : Running(R), Valid(!R) { if (Valid) Running = true; } @@ -195,34 +183,29 @@ public: } }; -inline bool loggingInitialized( - const __sanitizer::atomic_sint32_t &LoggingStatus) XRAY_NEVER_INSTRUMENT { - return __sanitizer::atomic_load(&LoggingStatus, - __sanitizer::memory_order_acquire) == - XRayLogInitStatus::XRAY_LOG_INITIALIZED; -} - } // namespace -inline void writeNewBufferPreamble(pid_t Tid, timespec TS, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { +static void writeNewBufferPreamble(pid_t Tid, + timespec TS) XRAY_NEVER_INSTRUMENT { static constexpr int InitRecordsCount = 2; - std::aligned_storage<sizeof(MetadataRecord)>::type Records[InitRecordsCount]; + auto &TLD = getThreadLocalData(); + MetadataRecord Metadata[InitRecordsCount]; { // Write out a MetadataRecord to signify that this is the start of a new // buffer, associated with a particular thread, with a new CPU. For the // data, we have 15 bytes to squeeze as much information as we can. At this // point we only write down the following bytes: // - Thread ID (pid_t, 4 bytes) - auto &NewBuffer = *reinterpret_cast<MetadataRecord *>(&Records[0]); + auto &NewBuffer = Metadata[0]; NewBuffer.Type = uint8_t(RecordType::Metadata); NewBuffer.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewBuffer); std::memcpy(&NewBuffer.Data, &Tid, sizeof(pid_t)); } + // Also write the WalltimeMarker record. { static_assert(sizeof(time_t) <= 8, "time_t needs to be at most 8 bytes"); - auto &WalltimeMarker = *reinterpret_cast<MetadataRecord *>(&Records[1]); + auto &WalltimeMarker = Metadata[1]; WalltimeMarker.Type = uint8_t(RecordType::Metadata); WalltimeMarker.RecordKind = uint8_t(MetadataRecord::RecordKinds::WalltimeMarker); @@ -235,26 +218,48 @@ inline void writeNewBufferPreamble(pid_t Tid, timespec TS, std::memcpy(WalltimeMarker.Data, &Seconds, sizeof(Seconds)); std::memcpy(WalltimeMarker.Data + sizeof(Seconds), &Micros, sizeof(Micros)); } - std::memcpy(MemPtr, Records, sizeof(MetadataRecord) * InitRecordsCount); - MemPtr += sizeof(MetadataRecord) * InitRecordsCount; - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + if (TLD.BQ == nullptr || TLD.BQ->finalizing()) + return; + std::memcpy(TLD.RecordPtr, Metadata, sizeof(Metadata)); + TLD.RecordPtr += sizeof(Metadata); + // Since we write out the extents as the first metadata record of the + // buffer, we need to write out the extents including the extents record. + __sanitizer::atomic_store(&TLD.Buffer.Extents->Size, sizeof(Metadata), + __sanitizer::memory_order_release); } inline void setupNewBuffer(int (*wall_clock_reader)( clockid_t, struct timespec *)) XRAY_NEVER_INSTRUMENT { - RecordPtr = static_cast<char *>(Buffer.Buffer); + auto &TLD = getThreadLocalData(); + auto &B = TLD.Buffer; + TLD.RecordPtr = static_cast<char *>(B.Buffer); pid_t Tid = syscall(SYS_gettid); timespec TS{0, 0}; // This is typically clock_gettime, but callers have injection ability. wall_clock_reader(CLOCK_MONOTONIC, &TS); - writeNewBufferPreamble(Tid, TS, RecordPtr); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + writeNewBufferPreamble(Tid, TS); + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; +} + +static void incrementExtents(size_t Add) { + auto &TLD = getThreadLocalData(); + __sanitizer::atomic_fetch_add(&TLD.Buffer.Extents->Size, Add, + __sanitizer::memory_order_acq_rel); } -inline void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { +static void decrementExtents(size_t Subtract) { + auto &TLD = getThreadLocalData(); + __sanitizer::atomic_fetch_sub(&TLD.Buffer.Extents->Size, Subtract, + __sanitizer::memory_order_acq_rel); +} + +inline void writeNewCPUIdMetadata(uint16_t CPU, + uint64_t TSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); MetadataRecord NewCPUId; NewCPUId.Type = uint8_t(RecordType::Metadata); NewCPUId.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewCPUId); @@ -265,34 +270,15 @@ inline void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC, // Total = 10 bytes. std::memcpy(&NewCPUId.Data, &CPU, sizeof(CPU)); std::memcpy(&NewCPUId.Data[sizeof(CPU)], &TSC, sizeof(TSC)); - std::memcpy(MemPtr, &NewCPUId, sizeof(MetadataRecord)); - MemPtr += sizeof(MetadataRecord); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; -} - -inline void writeNewCPUIdMetadata(uint16_t CPU, - uint64_t TSC) XRAY_NEVER_INSTRUMENT { - writeNewCPUIdMetadata(CPU, TSC, RecordPtr); -} - -inline void writeEOBMetadata(char *&MemPtr) XRAY_NEVER_INSTRUMENT { - MetadataRecord EOBMeta; - EOBMeta.Type = uint8_t(RecordType::Metadata); - EOBMeta.RecordKind = uint8_t(MetadataRecord::RecordKinds::EndOfBuffer); - // For now we don't write any bytes into the Data field. - std::memcpy(MemPtr, &EOBMeta, sizeof(MetadataRecord)); - MemPtr += sizeof(MetadataRecord); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + std::memcpy(TLD.RecordPtr, &NewCPUId, sizeof(MetadataRecord)); + TLD.RecordPtr += sizeof(MetadataRecord); + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + incrementExtents(sizeof(MetadataRecord)); } -inline void writeEOBMetadata() XRAY_NEVER_INSTRUMENT { - writeEOBMetadata(RecordPtr); -} - -inline void writeTSCWrapMetadata(uint64_t TSC, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { +inline void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); MetadataRecord TSCWrap; TSCWrap.Type = uint8_t(RecordType::Metadata); TSCWrap.RecordKind = uint8_t(MetadataRecord::RecordKinds::TSCWrap); @@ -301,58 +287,67 @@ inline void writeTSCWrapMetadata(uint64_t TSC, // - Full TSC (uint64_t, 8 bytes) // Total = 8 bytes. std::memcpy(&TSCWrap.Data, &TSC, sizeof(TSC)); - std::memcpy(MemPtr, &TSCWrap, sizeof(MetadataRecord)); - MemPtr += sizeof(MetadataRecord); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + std::memcpy(TLD.RecordPtr, &TSCWrap, sizeof(MetadataRecord)); + TLD.RecordPtr += sizeof(MetadataRecord); + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + incrementExtents(sizeof(MetadataRecord)); } -inline void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT { - writeTSCWrapMetadata(TSC, RecordPtr); +// Call Argument metadata records store the arguments to a function in the +// order of their appearance; holes are not supported by the buffer format. +static inline void writeCallArgumentMetadata(uint64_t A) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); + MetadataRecord CallArg; + CallArg.Type = uint8_t(RecordType::Metadata); + CallArg.RecordKind = uint8_t(MetadataRecord::RecordKinds::CallArgument); + + std::memcpy(CallArg.Data, &A, sizeof(A)); + std::memcpy(TLD.RecordPtr, &CallArg, sizeof(MetadataRecord)); + TLD.RecordPtr += sizeof(MetadataRecord); + incrementExtents(sizeof(MetadataRecord)); } -inline void writeFunctionRecord(int FuncId, uint32_t TSCDelta, - XRayEntryType EntryType, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { - std::aligned_storage<sizeof(FunctionRecord), alignof(FunctionRecord)>::type - AlignedFuncRecordBuffer; - auto &FuncRecord = - *reinterpret_cast<FunctionRecord *>(&AlignedFuncRecordBuffer); +static inline void +writeFunctionRecord(int FuncId, uint32_t TSCDelta, + XRayEntryType EntryType) XRAY_NEVER_INSTRUMENT { + FunctionRecord FuncRecord; FuncRecord.Type = uint8_t(RecordType::Function); // Only take 28 bits of the function id. FuncRecord.FuncId = FuncId & ~(0x0F << 28); FuncRecord.TSCDelta = TSCDelta; + auto &TLD = getThreadLocalData(); switch (EntryType) { case XRayEntryType::ENTRY: - ++NumConsecutiveFnEnters; + ++TLD.NumConsecutiveFnEnters; FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); break; case XRayEntryType::LOG_ARGS_ENTRY: // We should not rewind functions with logged args. - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); break; case XRayEntryType::EXIT: // If we've decided to log the function exit, we will never erase the log // before it. - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionExit); break; case XRayEntryType::TAIL: // If we just entered the function we're tail exiting from or erased every // invocation since then, this function entry tail pair is a candidate to // be erased when the child function exits. - if (NumConsecutiveFnEnters > 0) { - ++NumTailCalls; - NumConsecutiveFnEnters = 0; + if (TLD.NumConsecutiveFnEnters > 0) { + ++TLD.NumTailCalls; + TLD.NumConsecutiveFnEnters = 0; } else { // We will never be able to erase this tail call since we have logged // something in between the function entry and tail exit. - NumTailCalls = 0; - NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + TLD.NumConsecutiveFnEnters = 0; } FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionTailExit); @@ -370,8 +365,9 @@ inline void writeFunctionRecord(int FuncId, uint32_t TSCDelta, } } - std::memcpy(MemPtr, &AlignedFuncRecordBuffer, sizeof(FunctionRecord)); - MemPtr += sizeof(FunctionRecord); + std::memcpy(TLD.RecordPtr, &FuncRecord, sizeof(FunctionRecord)); + TLD.RecordPtr += sizeof(FunctionRecord); + incrementExtents(sizeof(FunctionRecord)); } static uint64_t thresholdTicks() { @@ -387,23 +383,21 @@ static uint64_t thresholdTicks() { // "Function Entry" record and any "Tail Call Exit" records after that. static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, uint64_t &LastFunctionEntryTSC, int32_t FuncId) { - using AlignedFuncStorage = - std::aligned_storage<sizeof(FunctionRecord), - alignof(FunctionRecord)>::type; - RecordPtr -= FunctionRecSize; - AlignedFuncStorage AlignedFuncRecordBuffer; - const auto &FuncRecord = *reinterpret_cast<FunctionRecord *>( - std::memcpy(&AlignedFuncRecordBuffer, RecordPtr, FunctionRecSize)); + auto &TLD = getThreadLocalData(); + TLD.RecordPtr -= FunctionRecSize; + decrementExtents(FunctionRecSize); + FunctionRecord FuncRecord; + std::memcpy(&FuncRecord, TLD.RecordPtr, FunctionRecSize); assert(FuncRecord.RecordKind == uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && "Expected to find function entry recording when rewinding."); assert(FuncRecord.FuncId == (FuncId & ~(0x0F << 28)) && "Expected matching function id when rewinding Exit"); - --NumConsecutiveFnEnters; + --TLD.NumConsecutiveFnEnters; LastTSC -= FuncRecord.TSCDelta; // We unwound one call. Update the state and return without writing a log. - if (NumConsecutiveFnEnters != 0) { + if (TLD.NumConsecutiveFnEnters != 0) { LastFunctionEntryTSC -= FuncRecord.TSCDelta; return; } @@ -413,22 +407,19 @@ static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, // exited from via this exit. LastFunctionEntryTSC = 0; auto RewindingTSC = LastTSC; - auto RewindingRecordPtr = RecordPtr - FunctionRecSize; - while (NumTailCalls > 0) { - AlignedFuncStorage TailExitRecordBuffer; + auto RewindingRecordPtr = TLD.RecordPtr - FunctionRecSize; + while (TLD.NumTailCalls > 0) { // Rewind the TSC back over the TAIL EXIT record. - const auto &ExpectedTailExit = - *reinterpret_cast<FunctionRecord *>(std::memcpy( - &TailExitRecordBuffer, RewindingRecordPtr, FunctionRecSize)); + FunctionRecord ExpectedTailExit; + std::memcpy(&ExpectedTailExit, RewindingRecordPtr, FunctionRecSize); assert(ExpectedTailExit.RecordKind == uint8_t(FunctionRecord::RecordKinds::FunctionTailExit) && "Expected to find tail exit when rewinding."); RewindingRecordPtr -= FunctionRecSize; RewindingTSC -= ExpectedTailExit.TSCDelta; - AlignedFuncStorage FunctionEntryBuffer; - const auto &ExpectedFunctionEntry = *reinterpret_cast<FunctionRecord *>( - std::memcpy(&FunctionEntryBuffer, RewindingRecordPtr, FunctionRecSize)); + FunctionRecord ExpectedFunctionEntry; + std::memcpy(&ExpectedFunctionEntry, RewindingRecordPtr, FunctionRecSize); assert(ExpectedFunctionEntry.RecordKind == uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && "Expected to find function entry when rewinding tail call."); @@ -437,80 +428,87 @@ static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, // This tail call exceeded the threshold duration. It will not be erased. if ((TSC - RewindingTSC) >= thresholdTicks()) { - NumTailCalls = 0; + TLD.NumTailCalls = 0; return; } // We can erase a tail exit pair that we're exiting through since // its duration is under threshold. - --NumTailCalls; + --TLD.NumTailCalls; RewindingRecordPtr -= FunctionRecSize; RewindingTSC -= ExpectedFunctionEntry.TSCDelta; - RecordPtr -= 2 * FunctionRecSize; + TLD.RecordPtr -= 2 * FunctionRecSize; LastTSC = RewindingTSC; + decrementExtents(2 * FunctionRecSize); } } -inline bool releaseThreadLocalBuffer(BufferQueue *BQ) { - auto EC = BQ->releaseBuffer(Buffer); +inline bool releaseThreadLocalBuffer(BufferQueue &BQArg) { + auto &TLD = getThreadLocalData(); + auto EC = BQArg.releaseBuffer(TLD.Buffer); if (EC != BufferQueue::ErrorCode::Ok) { - Report("Failed to release buffer at %p; error=%s\n", Buffer.Buffer, + Report("Failed to release buffer at %p; error=%s\n", TLD.Buffer.Buffer, BufferQueue::getErrorString(EC)); return false; } return true; } -inline bool prepareBuffer(int (*wall_clock_reader)(clockid_t, +inline bool prepareBuffer(uint64_t TSC, unsigned char CPU, + int (*wall_clock_reader)(clockid_t, struct timespec *), size_t MaxSize) XRAY_NEVER_INSTRUMENT { - char *BufferStart = static_cast<char *>(Buffer.Buffer); - if ((RecordPtr + MaxSize) > (BufferStart + Buffer.Size - MetadataRecSize)) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + auto &TLD = getThreadLocalData(); + char *BufferStart = static_cast<char *>(TLD.Buffer.Buffer); + if ((TLD.RecordPtr + MaxSize) > (BufferStart + TLD.Buffer.Size)) { + if (!releaseThreadLocalBuffer(*TLD.BQ)) return false; - auto EC = LocalBQ->getBuffer(Buffer); + auto EC = TLD.BQ->getBuffer(TLD.Buffer); if (EC != BufferQueue::ErrorCode::Ok) { Report("Failed to acquire a buffer; error=%s\n", BufferQueue::getErrorString(EC)); return false; } setupNewBuffer(wall_clock_reader); + + // Always write the CPU metadata as the first record in the buffer. + writeNewCPUIdMetadata(CPU, TSC); } return true; } -inline bool isLogInitializedAndReady( - std::shared_ptr<BufferQueue> &LocalBQ, uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, - struct timespec *)) XRAY_NEVER_INSTRUMENT { +inline bool +isLogInitializedAndReady(BufferQueue *LBQ, uint64_t TSC, unsigned char CPU, + int (*wall_clock_reader)(clockid_t, struct timespec *)) + XRAY_NEVER_INSTRUMENT { // Bail out right away if logging is not initialized yet. // We should take the opportunity to release the buffer though. auto Status = __sanitizer::atomic_load(&LoggingStatus, __sanitizer::memory_order_acquire); + auto &TLD = getThreadLocalData(); if (Status != XRayLogInitStatus::XRAY_LOG_INITIALIZED) { - if (RecordPtr != nullptr && + if (TLD.RecordPtr != nullptr && (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || Status == XRayLogInitStatus::XRAY_LOG_FINALIZED)) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + if (!releaseThreadLocalBuffer(*LBQ)) return false; - RecordPtr = nullptr; - LocalBQ = nullptr; + TLD.RecordPtr = nullptr; return false; } return false; } - if (!loggingInitialized(LoggingStatus) || LocalBQ->finalizing()) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + if (__sanitizer::atomic_load(&LoggingStatus, + __sanitizer::memory_order_acquire) != + XRayLogInitStatus::XRAY_LOG_INITIALIZED || + LBQ->finalizing()) { + if (!releaseThreadLocalBuffer(*LBQ)) return false; - RecordPtr = nullptr; + TLD.RecordPtr = nullptr; } - if (Buffer.Buffer == nullptr) { - auto EC = LocalBQ->getBuffer(Buffer); + if (TLD.Buffer.Buffer == nullptr) { + auto EC = LBQ->getBuffer(TLD.Buffer); if (EC != BufferQueue::ErrorCode::Ok) { auto LS = __sanitizer::atomic_load(&LoggingStatus, __sanitizer::memory_order_acquire); @@ -522,51 +520,100 @@ inline bool isLogInitializedAndReady( } setupNewBuffer(wall_clock_reader); + + // Always write the CPU metadata as the first record in the buffer. + writeNewCPUIdMetadata(CPU, TSC); } - if (CurrentCPU == std::numeric_limits<uint16_t>::max()) { + if (TLD.CurrentCPU == std::numeric_limits<uint16_t>::max()) { // This means this is the first CPU this thread has ever run on. We set // the current CPU and record this as the first TSC we've seen. - CurrentCPU = CPU; + TLD.CurrentCPU = CPU; writeNewCPUIdMetadata(CPU, TSC); } return true; } // namespace __xray_fdr_internal +// Compute the TSC difference between the time of measurement and the previous +// event. There are a few interesting situations we need to account for: +// +// - The thread has migrated to a different CPU. If this is the case, then +// we write down the following records: +// +// 1. A 'NewCPUId' Metadata record. +// 2. A FunctionRecord with a 0 for the TSCDelta field. +// +// - The TSC delta is greater than the 32 bits we can store in a +// FunctionRecord. In this case we write down the following records: +// +// 1. A 'TSCWrap' Metadata record. +// 2. A FunctionRecord with a 0 for the TSCDelta field. +// +// - The TSC delta is representable within the 32 bits we can store in a +// FunctionRecord. In this case we write down just a FunctionRecord with +// the correct TSC delta. +inline uint32_t writeCurrentCPUTSC(ThreadLocalData &TLD, uint64_t TSC, + uint8_t CPU) { + if (CPU != TLD.CurrentCPU) { + // We've moved to a new CPU. + writeNewCPUIdMetadata(CPU, TSC); + return 0; + } + // If the delta is greater than the range for a uint32_t, then we write out + // the TSC wrap metadata entry with the full TSC, and the TSC for the + // function record be 0. + uint64_t Delta = TSC - TLD.LastTSC; + if (Delta <= std::numeric_limits<uint32_t>::max()) + return Delta; + + writeTSCWrapMetadata(TSC); + return 0; +} + inline void endBufferIfFull() XRAY_NEVER_INSTRUMENT { - auto BufferStart = static_cast<char *>(Buffer.Buffer); - if ((RecordPtr + MetadataRecSize) - BufferStart == MetadataRecSize) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + auto &TLD = getThreadLocalData(); + auto BufferStart = static_cast<char *>(TLD.Buffer.Buffer); + if ((TLD.RecordPtr + MetadataRecSize) - BufferStart <= + ptrdiff_t{MetadataRecSize}) { + if (!releaseThreadLocalBuffer(*TLD.BQ)) return; - RecordPtr = nullptr; + TLD.RecordPtr = nullptr; } } -inline void processFunctionHook( - int32_t FuncId, XRayEntryType Entry, uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, struct timespec *), - __sanitizer::atomic_sint32_t &LoggingStatus, - const std::shared_ptr<BufferQueue> &BQ) XRAY_NEVER_INSTRUMENT { +thread_local volatile bool Running = false; + +/// Here's where the meat of the processing happens. The writer captures +/// function entry, exit and tail exit points with a time and will create +/// TSCWrap, NewCPUId and Function records as necessary. The writer might +/// walk backward through its buffer and erase trivial functions to avoid +/// polluting the log and may use the buffer queue to obtain or release a +/// buffer. +inline void processFunctionHook(int32_t FuncId, XRayEntryType Entry, + uint64_t TSC, unsigned char CPU, uint64_t Arg1, + int (*wall_clock_reader)(clockid_t, + struct timespec *), + BufferQueue *BQ) XRAY_NEVER_INSTRUMENT { // Prevent signal handler recursion, so in case we're already in a log writing // mode and the signal handler comes in (and is also instrumented) then we // don't want to be clobbering potentially partial writes already happening in // the thread. We use a simple thread_local latch to only allow one on-going // handleArg0 to happen at any given time. - thread_local bool Running = false; RecursionGuard Guard{Running}; if (!Guard) { assert(Running == true && "RecursionGuard is buggy!"); return; } + auto &TLD = getThreadLocalData(); + // In case the reference has been cleaned up before, we make sure we // initialize it to the provided BufferQueue. - if (LocalBQ == nullptr) - LocalBQ = BQ; + if (TLD.BQ == nullptr) + TLD.BQ = BQ; - if (!isLogInitializedAndReady(LocalBQ, TSC, CPU, wall_clock_reader)) + if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, wall_clock_reader)) return; // Before we go setting up writing new function entries, we need to be really @@ -579,10 +626,10 @@ inline void processFunctionHook( // - The least number of bytes we will ever write is 8 // (sizeof(FunctionRecord)) only if the delta between the previous entry // and this entry is within 32 bits. - // - The most number of bytes we will ever write is 8 + 16 = 24. This is - // computed by: + // - The most number of bytes we will ever write is 8 + 16 + 16 = 40. + // This is computed by: // - // sizeof(FunctionRecord) + sizeof(MetadataRecord) + // MaxSize = sizeof(FunctionRecord) + 2 * sizeof(MetadataRecord) // // These arise in the following cases: // @@ -596,77 +643,39 @@ inline void processFunctionHook( // FunctionRecord. // 3. When we learn about a new CPU ID, we need to write down a "new cpu // id" MetadataRecord before writing out the actual FunctionRecord. + // 4. The second MetadataRecord is the optional function call argument. // - // - An End-of-Buffer (EOB) MetadataRecord is 16 bytes. - // - // So the math we need to do is to determine whether writing 24 bytes past the - // current pointer leaves us with enough bytes to write the EOB - // MetadataRecord. If we don't have enough space after writing as much as 24 - // bytes in the end of the buffer, we need to write out the EOB, get a new - // Buffer, set it up properly before doing any further writing. - // - if (!prepareBuffer(wall_clock_reader, FunctionRecSize + MetadataRecSize)) { - LocalBQ = nullptr; + // So the math we need to do is to determine whether writing 40 bytes past the + // current pointer exceeds the buffer's maximum size. If we don't have enough + // space to write 40 bytes in the buffer, we need get a new Buffer, set it up + // properly before doing any further writing. + size_t MaxSize = FunctionRecSize + 2 * MetadataRecSize; + if (!prepareBuffer(TSC, CPU, wall_clock_reader, MaxSize)) { + TLD.BQ = nullptr; return; } - // By this point, we are now ready to write at most 24 bytes (one metadata - // record and one function record). - assert((RecordPtr + (MetadataRecSize + FunctionRecSize)) - - static_cast<char *>(Buffer.Buffer) >= + // By this point, we are now ready to write up to 40 bytes (explained above). + assert((TLD.RecordPtr + MaxSize) - static_cast<char *>(TLD.Buffer.Buffer) >= static_cast<ptrdiff_t>(MetadataRecSize) && "Misconfigured BufferQueue provided; Buffer size not large enough."); - // Here we compute the TSC Delta. There are a few interesting situations we - // need to account for: - // - // - The thread has migrated to a different CPU. If this is the case, then - // we write down the following records: - // - // 1. A 'NewCPUId' Metadata record. - // 2. A FunctionRecord with a 0 for the TSCDelta field. - // - // - The TSC delta is greater than the 32 bits we can store in a - // FunctionRecord. In this case we write down the following records: - // - // 1. A 'TSCWrap' Metadata record. - // 2. A FunctionRecord with a 0 for the TSCDelta field. - // - // - The TSC delta is representable within the 32 bits we can store in a - // FunctionRecord. In this case we write down just a FunctionRecord with - // the correct TSC delta. - // - uint32_t RecordTSCDelta = 0; - if (CPU != CurrentCPU) { - // We've moved to a new CPU. - writeNewCPUIdMetadata(CPU, TSC); - } else { - // If the delta is greater than the range for a uint32_t, then we write out - // the TSC wrap metadata entry with the full TSC, and the TSC for the - // function record be 0. - auto Delta = TSC - LastTSC; - if (Delta > (1ULL << 32) - 1) - writeTSCWrapMetadata(TSC); - else - RecordTSCDelta = Delta; - } - - LastTSC = TSC; - CurrentCPU = CPU; + auto RecordTSCDelta = writeCurrentCPUTSC(TLD, TSC, CPU); + TLD.LastTSC = TSC; + TLD.CurrentCPU = CPU; switch (Entry) { case XRayEntryType::ENTRY: case XRayEntryType::LOG_ARGS_ENTRY: // Update the thread local state for the next invocation. - LastFunctionEntryTSC = TSC; + TLD.LastFunctionEntryTSC = TSC; break; case XRayEntryType::TAIL: - break; case XRayEntryType::EXIT: // Break out and write the exit record if we can't erase any functions. - if (NumConsecutiveFnEnters == 0 || - (TSC - LastFunctionEntryTSC) >= thresholdTicks()) + if (TLD.NumConsecutiveFnEnters == 0 || + (TSC - TLD.LastFunctionEntryTSC) >= thresholdTicks()) break; - rewindRecentCall(TSC, LastTSC, LastFunctionEntryTSC, FuncId); + rewindRecentCall(TSC, TLD.LastTSC, TLD.LastFunctionEntryTSC, FuncId); return; // without writing log. case XRayEntryType::CUSTOM_EVENT: { // This is a bug in patching, so we'll report it once and move on. @@ -681,7 +690,9 @@ inline void processFunctionHook( } } - writeFunctionRecord(FuncId, RecordTSCDelta, Entry, RecordPtr); + writeFunctionRecord(FuncId, RecordTSCDelta, Entry); + if (Entry == XRayEntryType::LOG_ARGS_ENTRY) + writeCallArgumentMetadata(Arg1); // If we've exhausted the buffer by this time, we then release the buffer to // make sure that other threads may start using this buffer. diff --git a/contrib/compiler-rt/lib/xray/xray_flags.h b/contrib/compiler-rt/lib/xray/xray_flags.h index f4e30283b8de..3ed5b8844cb4 100644 --- a/contrib/compiler-rt/lib/xray/xray_flags.h +++ b/contrib/compiler-rt/lib/xray/xray_flags.h @@ -16,6 +16,7 @@ #define XRAY_FLAGS_H #include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_internal_defs.h" namespace __xray { diff --git a/contrib/compiler-rt/lib/xray/xray_flags.inc b/contrib/compiler-rt/lib/xray/xray_flags.inc index 7ddce78eb413..29f1fce7d7f4 100644 --- a/contrib/compiler-rt/lib/xray/xray_flags.inc +++ b/contrib/compiler-rt/lib/xray/xray_flags.inc @@ -16,12 +16,34 @@ XRAY_FLAG(bool, patch_premain, false, "Whether to patch instrumentation points before main.") -XRAY_FLAG(bool, xray_naive_log, true, - "Whether to install the naive log implementation.") XRAY_FLAG(const char *, xray_logfile_base, "xray-log.", "Filename base for the xray logfile.") +XRAY_FLAG(const char *, xray_mode, "", "Mode to install by default.") +XRAY_FLAG(uptr, xray_page_size_override, 0, + "Override the default page size for the system, in bytes. The size " + "should be a power-of-two.") + +// Basic (Naive) Mode logging options. +XRAY_FLAG(bool, xray_naive_log, false, + "DEPRECATED: Use xray_mode=xray-basic instead.") +XRAY_FLAG(int, xray_naive_log_func_duration_threshold_us, 5, + "Naive logging will try to skip functions that execute for fewer " + "microseconds than this threshold.") +XRAY_FLAG(int, xray_naive_log_max_stack_depth, 64, + "Naive logging will keep track of at most this deep a call stack, " + "any more and the recordings will be droppped.") +XRAY_FLAG(int, xray_naive_log_thread_buffer_size, 1024, + "The number of entries to keep on a per-thread buffer.") + +// FDR (Flight Data Recorder) Mode logging options. XRAY_FLAG(bool, xray_fdr_log, false, - "Whether to install the flight data recorder logging implementation.") + "DEPRECATED: Use xray_mode=xray-fdr instead.") XRAY_FLAG(int, xray_fdr_log_func_duration_threshold_us, 5, "FDR logging will try to skip functions that execute for fewer " "microseconds than this threshold.") +XRAY_FLAG(int, xray_fdr_log_grace_period_us, 0, + "DEPRECATED: use xray_fdr_log_grace_period_ms instead.") +XRAY_FLAG(int, xray_fdr_log_grace_period_ms, 100, + "FDR logging will wait this much time in microseconds before " + "actually flushing the log; this gives a chance for threads to " + "notice that the log has been finalized and clean up.") diff --git a/contrib/compiler-rt/lib/xray/xray_init.cc b/contrib/compiler-rt/lib/xray/xray_init.cc index aa660baa9920..11892cb8b7a3 100644 --- a/contrib/compiler-rt/lib/xray/xray_init.cc +++ b/contrib/compiler-rt/lib/xray/xray_init.cc @@ -44,12 +44,31 @@ __sanitizer::atomic_uint8_t XRayInitialized{0}; __sanitizer::SpinMutex XRayInstrMapMutex; XRaySledMap XRayInstrMap; +// Global flag to determine whether the flags have been initialized. +__sanitizer::atomic_uint8_t XRayFlagsInitialized{0}; + +// A mutex to allow only one thread to initialize the XRay data structures. +__sanitizer::SpinMutex XRayInitMutex; + // __xray_init() will do the actual loading of the current process' memory map // and then proceed to look for the .xray_instr_map section/segment. void __xray_init() XRAY_NEVER_INSTRUMENT { - initializeFlags(); + __sanitizer::SpinMutexLock Guard(&XRayInitMutex); + // Short-circuit if we've already initialized XRay before. + if (__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) + return; + + if (!__sanitizer::atomic_load(&XRayFlagsInitialized, + __sanitizer::memory_order_acquire)) { + initializeFlags(); + __sanitizer::atomic_store(&XRayFlagsInitialized, true, + __sanitizer::memory_order_release); + } + if (__start_xray_instr_map == nullptr) { - Report("XRay instrumentation map missing. Not initializing XRay.\n"); + if (Verbosity()) + Report("XRay instrumentation map missing. Not initializing XRay.\n"); return; } @@ -63,9 +82,21 @@ void __xray_init() XRAY_NEVER_INSTRUMENT { __sanitizer::atomic_store(&XRayInitialized, true, __sanitizer::memory_order_release); +#ifndef XRAY_NO_PREINIT if (flags()->patch_premain) __xray_patch(); +#endif } +#if !defined(XRAY_NO_PREINIT) && SANITIZER_CAN_USE_PREINIT_ARRAY +// Only add the preinit array initialization if the sanitizers can. __attribute__((section(".preinit_array"), used)) void (*__local_xray_preinit)(void) = __xray_init; +#else +// If we cannot use the .preinit_array section, we should instead use dynamic +// initialisation. +static bool UNUSED __local_xray_dyninit = [] { + __xray_init(); + return true; +}(); +#endif diff --git a/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc b/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc index 83aecfaf7700..a27ffbcbd12e 100644 --- a/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc +++ b/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc @@ -16,80 +16,85 @@ //===----------------------------------------------------------------------===// #include <cassert> +#include <cstring> +#include <errno.h> #include <fcntl.h> -#include <mutex> +#include <pthread.h> #include <sys/stat.h> #include <sys/syscall.h> #include <sys/types.h> -#include <thread> +#include <time.h> #include <unistd.h> +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_libc.h" #include "xray/xray_records.h" #include "xray_defs.h" #include "xray_flags.h" +#include "xray_inmemory_log.h" #include "xray_interface_internal.h" #include "xray_tsc.h" #include "xray_utils.h" -// __xray_InMemoryRawLog will use a thread-local aligned buffer capped to a -// certain size (32kb by default) and use it as if it were a circular buffer for -// events. We store simple fixed-sized entries in the log for external analysis. - -extern "C" { -void __xray_InMemoryRawLog(int32_t FuncId, - XRayEntryType Type) XRAY_NEVER_INSTRUMENT; -} - namespace __xray { -std::mutex LogMutex; +__sanitizer::SpinMutex LogMutex; -class ThreadExitFlusher { - int Fd; - XRayRecord *Start; - size_t &Offset; +// We use elements of this type to record the entry TSC of every function ID we +// see as we're tracing a particular thread's execution. +struct alignas(16) StackEntry { + int32_t FuncId; + uint16_t Type; + uint8_t CPU; + uint8_t Padding; + uint64_t TSC; +}; -public: - explicit ThreadExitFlusher(int Fd, XRayRecord *Start, - size_t &Offset) XRAY_NEVER_INSTRUMENT - : Fd(Fd), - Start(Start), - Offset(Offset) {} +static_assert(sizeof(StackEntry) == 16, "Wrong size for StackEntry"); - ~ThreadExitFlusher() XRAY_NEVER_INSTRUMENT { - std::lock_guard<std::mutex> L(LogMutex); - if (Fd > 0 && Start != nullptr) { - retryingWriteAll(Fd, reinterpret_cast<char *>(Start), - reinterpret_cast<char *>(Start + Offset)); - // Because this thread's exit could be the last one trying to write to the - // file and that we're not able to close out the file properly, we sync - // instead and hope that the pending writes are flushed as the thread - // exits. - fsync(Fd); - } - } +struct alignas(64) ThreadLocalData { + void *InMemoryBuffer = nullptr; + size_t BufferSize = 0; + size_t BufferOffset = 0; + void *ShadowStack = nullptr; + size_t StackSize = 0; + size_t StackEntries = 0; + int Fd = -1; + pid_t TID = 0; }; -} // namespace __xray +static pthread_key_t PThreadKey; + +static __sanitizer::atomic_uint8_t BasicInitialized{0}; + +BasicLoggingOptions GlobalOptions; -using namespace __xray; +thread_local volatile bool RecursionGuard = false; + +static uint64_t thresholdTicks() XRAY_NEVER_INSTRUMENT { + static uint64_t TicksPerSec = probeRequiredCPUFeatures() + ? getTSCFrequency() + : __xray::NanosecondsPerSecond; + static const uint64_t ThresholdTicks = + TicksPerSec * GlobalOptions.DurationFilterMicros / 1000000; + return ThresholdTicks; +} -static int __xray_OpenLogFile() XRAY_NEVER_INSTRUMENT { +static int openLogFile() XRAY_NEVER_INSTRUMENT { int F = getLogFD(); if (F == -1) return -1; // Test for required CPU features and cache the cycle frequency static bool TSCSupported = probeRequiredCPUFeatures(); - static uint64_t CycleFrequency = TSCSupported ? getTSCFrequency() - : __xray::NanosecondsPerSecond; + static uint64_t CycleFrequency = + TSCSupported ? getTSCFrequency() : __xray::NanosecondsPerSecond; // Since we're here, we get to write the header. We set it up so that the // header will only be written once, at the start, and let the threads // logging do writes which just append. XRayFileHeader Header; - Header.Version = 1; + Header.Version = 2; // Version 2 includes tail exit records. Header.Type = FileTypes::NAIVE_LOG; Header.CycleFrequency = CycleFrequency; @@ -102,47 +107,210 @@ static int __xray_OpenLogFile() XRAY_NEVER_INSTRUMENT { return F; } +int getGlobalFd() XRAY_NEVER_INSTRUMENT { + static int Fd = openLogFile(); + return Fd; +} + +ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { + thread_local ThreadLocalData TLD; + thread_local bool UNUSED TOnce = [] { + if (GlobalOptions.ThreadBufferSize == 0) { + if (__sanitizer::Verbosity()) + Report("Not initializing TLD since ThreadBufferSize == 0.\n"); + return false; + } + TLD.TID = __sanitizer::GetTid(); + pthread_setspecific(PThreadKey, &TLD); + TLD.Fd = getGlobalFd(); + TLD.InMemoryBuffer = reinterpret_cast<XRayRecord *>( + InternalAlloc(sizeof(XRayRecord) * GlobalOptions.ThreadBufferSize, + nullptr, alignof(XRayRecord))); + TLD.BufferSize = GlobalOptions.ThreadBufferSize; + TLD.BufferOffset = 0; + if (GlobalOptions.MaxStackDepth == 0) { + if (__sanitizer::Verbosity()) + Report("Not initializing the ShadowStack since MaxStackDepth == 0.\n"); + TLD.StackSize = 0; + TLD.StackEntries = 0; + TLD.ShadowStack = nullptr; + return false; + } + TLD.ShadowStack = reinterpret_cast<StackEntry *>( + InternalAlloc(sizeof(StackEntry) * GlobalOptions.MaxStackDepth, nullptr, + alignof(StackEntry))); + TLD.StackSize = GlobalOptions.MaxStackDepth; + TLD.StackEntries = 0; + if (__sanitizer::Verbosity() >= 2) { + static auto UNUSED Once = [] { + auto ticks = thresholdTicks(); + Report("Ticks threshold: %d\n", ticks); + return false; + }(); + } + return false; + }(); + return TLD; +} + template <class RDTSC> -void __xray_InMemoryRawLog(int32_t FuncId, XRayEntryType Type, - RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { - using Buffer = - std::aligned_storage<sizeof(XRayRecord), alignof(XRayRecord)>::type; - static constexpr size_t BuffLen = 1024; - thread_local static Buffer InMemoryBuffer[BuffLen] = {}; - thread_local static size_t Offset = 0; - static int Fd = __xray_OpenLogFile(); +void InMemoryRawLog(int32_t FuncId, XRayEntryType Type, + RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); + auto &InMemoryBuffer = TLD.InMemoryBuffer; + int Fd = getGlobalFd(); if (Fd == -1) return; - thread_local __xray::ThreadExitFlusher Flusher( - Fd, reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer), Offset); - thread_local pid_t TId = syscall(SYS_gettid); - // First we get the useful data, and stuff it into the already aligned buffer - // through a pointer offset. - auto &R = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer)[Offset]; + // Use a simple recursion guard, to handle cases where we're already logging + // and for one reason or another, this function gets called again in the same + // thread. + if (RecursionGuard) + return; + RecursionGuard = true; + auto ExitGuard = __sanitizer::at_scope_exit([] { RecursionGuard = false; }); + + uint8_t CPU = 0; + uint64_t TSC = ReadTSC(CPU); + + switch (Type) { + case XRayEntryType::ENTRY: + case XRayEntryType::LOG_ARGS_ENTRY: { + // Short circuit if we've reached the maximum depth of the stack. + if (TLD.StackEntries++ >= TLD.StackSize) + return; + + // When we encounter an entry event, we keep track of the TSC and the CPU, + // and put it in the stack. + StackEntry E; + E.FuncId = FuncId; + E.CPU = CPU; + E.Type = Type; + E.TSC = TSC; + auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) + + (sizeof(StackEntry) * (TLD.StackEntries - 1)); + __sanitizer::internal_memcpy(StackEntryPtr, &E, sizeof(StackEntry)); + break; + } + case XRayEntryType::EXIT: + case XRayEntryType::TAIL: { + if (TLD.StackEntries == 0) + break; + + if (--TLD.StackEntries >= TLD.StackSize) + return; + + // When we encounter an exit event, we check whether all the following are + // true: + // + // - The Function ID is the same as the most recent entry in the stack. + // - The CPU is the same as the most recent entry in the stack. + // - The Delta of the TSCs is less than the threshold amount of time we're + // looking to record. + // + // If all of these conditions are true, we pop the stack and don't write a + // record and move the record offset back. + StackEntry StackTop; + auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) + + (sizeof(StackEntry) * TLD.StackEntries); + __sanitizer::internal_memcpy(&StackTop, StackEntryPtr, sizeof(StackEntry)); + if (StackTop.FuncId == FuncId && StackTop.CPU == CPU && + StackTop.TSC < TSC) { + auto Delta = TSC - StackTop.TSC; + if (Delta < thresholdTicks()) { + assert(TLD.BufferOffset > 0); + TLD.BufferOffset -= StackTop.Type == XRayEntryType::ENTRY ? 1 : 2; + return; + } + } + break; + } + default: + // Should be unreachable. + assert(false && "Unsupported XRayEntryType encountered."); + break; + } + + // First determine whether the delta between the function's enter record and + // the exit record is higher than the threshold. + __xray::XRayRecord R; R.RecordType = RecordTypes::NORMAL; - R.TSC = ReadTSC(R.CPU); - R.TId = TId; + R.CPU = CPU; + R.TSC = TSC; + R.TId = TLD.TID; R.Type = Type; R.FuncId = FuncId; - ++Offset; - if (Offset == BuffLen) { - std::lock_guard<std::mutex> L(LogMutex); + auto EntryPtr = static_cast<char *>(InMemoryBuffer) + + (sizeof(__xray::XRayRecord) * TLD.BufferOffset); + __sanitizer::internal_memcpy(EntryPtr, &R, sizeof(R)); + if (++TLD.BufferOffset == TLD.BufferSize) { + __sanitizer::SpinMutexLock L(&LogMutex); + auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer); + retryingWriteAll(Fd, reinterpret_cast<char *>(RecordBuffer), + reinterpret_cast<char *>(RecordBuffer + TLD.BufferOffset)); + TLD.BufferOffset = 0; + TLD.StackEntries = 0; + } +} + +template <class RDTSC> +void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1, + RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); + auto &InMemoryBuffer = TLD.InMemoryBuffer; + auto &Offset = TLD.BufferOffset; + const auto &BuffLen = TLD.BufferSize; + int Fd = getGlobalFd(); + if (Fd == -1) + return; + + // First we check whether there's enough space to write the data consecutively + // in the thread-local buffer. If not, we first flush the buffer before + // attempting to write the two records that must be consecutive. + if (Offset + 2 > BuffLen) { + __sanitizer::SpinMutexLock L(&LogMutex); auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer); retryingWriteAll(Fd, reinterpret_cast<char *>(RecordBuffer), reinterpret_cast<char *>(RecordBuffer + Offset)); Offset = 0; + TLD.StackEntries = 0; + } + + // Then we write the "we have an argument" record. + InMemoryRawLog(FuncId, Type, ReadTSC); + + if (RecursionGuard) + return; + RecursionGuard = true; + auto ExitGuard = __sanitizer::at_scope_exit([] { RecursionGuard = false; }); + + // And from here on write the arg payload. + __xray::XRayArgPayload R; + R.RecordType = RecordTypes::ARG_PAYLOAD; + R.FuncId = FuncId; + R.TId = TLD.TID; + R.Arg = Arg1; + auto EntryPtr = + &reinterpret_cast<__xray::XRayArgPayload *>(&InMemoryBuffer)[Offset]; + std::memcpy(EntryPtr, &R, sizeof(R)); + if (++Offset == BuffLen) { + __sanitizer::SpinMutexLock L(&LogMutex); + auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer); + retryingWriteAll(Fd, reinterpret_cast<char *>(RecordBuffer), + reinterpret_cast<char *>(RecordBuffer + Offset)); + Offset = 0; + TLD.StackEntries = 0; } } -void __xray_InMemoryRawLogRealTSC(int32_t FuncId, - XRayEntryType Type) XRAY_NEVER_INSTRUMENT { - __xray_InMemoryRawLog(FuncId, Type, __xray::readTSC); +void basicLoggingHandleArg0RealTSC(int32_t FuncId, + XRayEntryType Type) XRAY_NEVER_INSTRUMENT { + InMemoryRawLog(FuncId, Type, __xray::readTSC); } -void __xray_InMemoryEmulateTSC(int32_t FuncId, - XRayEntryType Type) XRAY_NEVER_INSTRUMENT { - __xray_InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT { +void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Type) + XRAY_NEVER_INSTRUMENT { + InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT { timespec TS; int result = clock_gettime(CLOCK_REALTIME, &TS); if (result != 0) { @@ -154,13 +322,150 @@ void __xray_InMemoryEmulateTSC(int32_t FuncId, }); } -static auto UNUSED Unused = [] { - auto UseRealTSC = probeRequiredCPUFeatures(); - if (!UseRealTSC) +void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Type, + uint64_t Arg1) XRAY_NEVER_INSTRUMENT { + InMemoryRawLogWithArg(FuncId, Type, Arg1, __xray::readTSC); +} + +void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Type, + uint64_t Arg1) XRAY_NEVER_INSTRUMENT { + InMemoryRawLogWithArg( + FuncId, Type, Arg1, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT { + timespec TS; + int result = clock_gettime(CLOCK_REALTIME, &TS); + if (result != 0) { + Report("clock_gettimg(2) return %d, errno=%d.", result, int(errno)); + TS = {0, 0}; + } + CPU = 0; + return TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec; + }); +} + +static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT { + ThreadLocalData &TLD = *reinterpret_cast<ThreadLocalData *>(P); + auto ExitGuard = __sanitizer::at_scope_exit([&TLD] { + // Clean up dynamic resources. + if (TLD.InMemoryBuffer) + InternalFree(TLD.InMemoryBuffer); + if (TLD.ShadowStack) + InternalFree(TLD.ShadowStack); + if (__sanitizer::Verbosity()) + Report("Cleaned up log for TID: %d\n", TLD.TID); + }); + + if (TLD.Fd == -1 || TLD.BufferOffset == 0) { + if (__sanitizer::Verbosity()) + Report("Skipping buffer for TID: %d; Fd = %d; Offset = %llu\n", TLD.TID, + TLD.Fd, TLD.BufferOffset); + return; + } + + { + __sanitizer::SpinMutexLock L(&LogMutex); + retryingWriteAll(TLD.Fd, reinterpret_cast<char *>(TLD.InMemoryBuffer), + reinterpret_cast<char *>(TLD.InMemoryBuffer) + + (sizeof(__xray::XRayRecord) * TLD.BufferOffset)); + } + + // Because this thread's exit could be the last one trying to write to + // the file and that we're not able to close out the file properly, we + // sync instead and hope that the pending writes are flushed as the + // thread exits. + fsync(TLD.Fd); +} + +XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, + void *Options, + size_t OptionsSize) XRAY_NEVER_INSTRUMENT { + static bool UNUSED Once = [] { + pthread_key_create(&PThreadKey, TLDDestructor); + return false; + }(); + + uint8_t Expected = 0; + if (!__sanitizer::atomic_compare_exchange_strong( + &BasicInitialized, &Expected, 1, __sanitizer::memory_order_acq_rel)) { + if (__sanitizer::Verbosity()) + Report("Basic logging already initialized.\n"); + return XRayLogInitStatus::XRAY_LOG_INITIALIZED; + } + + if (OptionsSize != sizeof(BasicLoggingOptions)) { + Report("Invalid options size, potential ABI mismatch; expected %d got %d", + sizeof(BasicLoggingOptions), OptionsSize); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } + + static auto UseRealTSC = probeRequiredCPUFeatures(); + if (!UseRealTSC && __sanitizer::Verbosity()) Report("WARNING: Required CPU features missing for XRay instrumentation, " "using emulation instead.\n"); - if (flags()->xray_naive_log) - __xray_set_handler(UseRealTSC ? __xray_InMemoryRawLogRealTSC - : __xray_InMemoryEmulateTSC); + + GlobalOptions = *reinterpret_cast<BasicLoggingOptions *>(Options); + __xray_set_handler_arg1(UseRealTSC ? basicLoggingHandleArg1RealTSC + : basicLoggingHandleArg1EmulateTSC); + __xray_set_handler(UseRealTSC ? basicLoggingHandleArg0RealTSC + : basicLoggingHandleArg0EmulateTSC); + __xray_remove_customevent_handler(); + + return XRayLogInitStatus::XRAY_LOG_INITIALIZED; +} + +XRayLogInitStatus basicLoggingFinalize() XRAY_NEVER_INSTRUMENT { + uint8_t Expected = 0; + if (!__sanitizer::atomic_compare_exchange_strong( + &BasicInitialized, &Expected, 0, __sanitizer::memory_order_acq_rel) && + __sanitizer::Verbosity()) + Report("Basic logging already finalized.\n"); + + // Nothing really to do aside from marking state of the global to be + // uninitialized. + + return XRayLogInitStatus::XRAY_LOG_FINALIZED; +} + +XRayLogFlushStatus basicLoggingFlush() XRAY_NEVER_INSTRUMENT { + // This really does nothing, since flushing the logs happen at the end of a + // thread's lifetime, or when the buffers are full. + return XRayLogFlushStatus::XRAY_LOG_FLUSHED; +} + +// This is a handler that, effectively, does nothing. +void basicLoggingHandleArg0Empty(int32_t, XRayEntryType) XRAY_NEVER_INSTRUMENT { +} + +bool basicLogDynamicInitializer() XRAY_NEVER_INSTRUMENT { + XRayLogImpl Impl{ + basicLoggingInit, + basicLoggingFinalize, + basicLoggingHandleArg0Empty, + basicLoggingFlush, + }; + auto RegistrationResult = __xray_log_register_mode("xray-basic", Impl); + if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && + __sanitizer::Verbosity()) + Report("Cannot register XRay Basic Mode to 'xray-basic'; error = %d\n", + RegistrationResult); + if (flags()->xray_naive_log || + !__sanitizer::internal_strcmp(flags()->xray_mode, "xray-basic")) { + __xray_set_log_impl(Impl); + BasicLoggingOptions Options; + Options.DurationFilterMicros = + flags()->xray_naive_log_func_duration_threshold_us; + Options.MaxStackDepth = flags()->xray_naive_log_max_stack_depth; + Options.ThreadBufferSize = flags()->xray_naive_log_thread_buffer_size; + __xray_log_init(flags()->xray_naive_log_thread_buffer_size, 0, &Options, + sizeof(BasicLoggingOptions)); + static auto UNUSED Once = [] { + static auto UNUSED &TLD = getThreadLocalData(); + __sanitizer::Atexit(+[] { TLDDestructor(&TLD); }); + return false; + }(); + } return true; -}(); +} + +} // namespace __xray + +static auto UNUSED Unused = __xray::basicLogDynamicInitializer(); diff --git a/contrib/compiler-rt/lib/xray/xray_inmemory_log.h b/contrib/compiler-rt/lib/xray/xray_inmemory_log.h new file mode 100644 index 000000000000..e4fcb8ca5ffd --- /dev/null +++ b/contrib/compiler-rt/lib/xray/xray_inmemory_log.h @@ -0,0 +1,44 @@ +//===-- xray_inmemory_log.h +//------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a function call tracing system. +// +//===----------------------------------------------------------------------===// +#ifndef XRAY_XRAY_INMEMORY_LOG_H +#define XRAY_XRAY_INMEMORY_LOG_H + +#include "xray/xray_log_interface.h" + +/// Basic (Naive) Mode +/// ================== +/// +/// This implementation hooks in through the XRay logging implementation +/// framework. The Basic Mode implementation will keep appending to a file as +/// soon as the thread-local buffers are full. It keeps minimal in-memory state +/// and does the minimum filtering required to keep log files smaller. + +namespace __xray { + +XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, + void *Options, size_t OptionsSize); +XRayLogInitStatus basicLoggingFinalize(); + +void basicLoggingHandleArg0RealTSC(int32_t FuncId, XRayEntryType Entry); +void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Entry); +void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Entry, + uint64_t Arg1); +void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Entry, + uint64_t Arg1); +XRayLogFlushStatus basicLoggingFlush(); +XRayLogInitStatus basicLoggingReset(); + +} // namespace __xray + +#endif // XRAY_XRAY_INMEMORY_LOG_H diff --git a/contrib/compiler-rt/lib/xray/xray_interface.cc b/contrib/compiler-rt/lib/xray/xray_interface.cc index 694d34c0102b..766313e85c58 100644 --- a/contrib/compiler-rt/lib/xray/xray_interface.cc +++ b/contrib/compiler-rt/lib/xray/xray_interface.cc @@ -23,12 +23,15 @@ #include "sanitizer_common/sanitizer_common.h" #include "xray_defs.h" +#include "xray_flags.h" + +extern __sanitizer::SpinMutex XRayInstrMapMutex; +extern __sanitizer::atomic_uint8_t XRayInitialized; +extern __xray::XRaySledMap XRayInstrMap; namespace __xray { #if defined(__x86_64__) -// FIXME: The actual length is 11 bytes. Why was length 12 passed to mprotect() -// ? static const int16_t cSledLength = 12; #elif defined(__aarch64__) static const int16_t cSledLength = 32; @@ -53,6 +56,10 @@ __sanitizer::atomic_uintptr_t XRayArgLogger{0}; // This is the function to call when we encounter a custom event log call. __sanitizer::atomic_uintptr_t XRayPatchedCustomEvent{0}; +// This is the global status to determine whether we are currently +// patching/unpatching. +__sanitizer::atomic_uint8_t XRayPatching{0}; + // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will undo // any successful mprotect(...) changes. This is used to make a page writeable // and executable, and upon destruction if it was successful in doing so returns @@ -88,85 +95,10 @@ public: } }; -} // namespace __xray - -extern __sanitizer::SpinMutex XRayInstrMapMutex; -extern __sanitizer::atomic_uint8_t XRayInitialized; -extern __xray::XRaySledMap XRayInstrMap; - -int __xray_set_handler(void (*entry)(int32_t, - XRayEntryType)) XRAY_NEVER_INSTRUMENT { - if (__sanitizer::atomic_load(&XRayInitialized, - __sanitizer::memory_order_acquire)) { - - __sanitizer::atomic_store(&__xray::XRayPatchedFunction, - reinterpret_cast<uintptr_t>(entry), - __sanitizer::memory_order_release); - return 1; - } - return 0; -} - -int __xray_set_customevent_handler(void (*entry)(void *, size_t)) - XRAY_NEVER_INSTRUMENT { - if (__sanitizer::atomic_load(&XRayInitialized, - __sanitizer::memory_order_acquire)) { - __sanitizer::atomic_store(&__xray::XRayPatchedCustomEvent, - reinterpret_cast<uintptr_t>(entry), - __sanitizer::memory_order_release); - return 1; - } - return 0; -} - - -int __xray_remove_handler() XRAY_NEVER_INSTRUMENT { - return __xray_set_handler(nullptr); -} - -int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT { - return __xray_set_customevent_handler(nullptr); -} - -__sanitizer::atomic_uint8_t XRayPatching{0}; - -using namespace __xray; - -// FIXME: Figure out whether we can move this class to sanitizer_common instead -// as a generic "scope guard". -template <class Function> class CleanupInvoker { - Function Fn; - -public: - explicit CleanupInvoker(Function Fn) XRAY_NEVER_INSTRUMENT : Fn(Fn) {} - CleanupInvoker(const CleanupInvoker &) XRAY_NEVER_INSTRUMENT = default; - CleanupInvoker(CleanupInvoker &&) XRAY_NEVER_INSTRUMENT = default; - CleanupInvoker & - operator=(const CleanupInvoker &) XRAY_NEVER_INSTRUMENT = delete; - CleanupInvoker &operator=(CleanupInvoker &&) XRAY_NEVER_INSTRUMENT = delete; - ~CleanupInvoker() XRAY_NEVER_INSTRUMENT { Fn(); } -}; - -template <class Function> -CleanupInvoker<Function> scopeCleanup(Function Fn) XRAY_NEVER_INSTRUMENT { - return CleanupInvoker<Function>{Fn}; -} - -inline bool patchSled(const XRaySledEntry &Sled, bool Enable, - int32_t FuncId) XRAY_NEVER_INSTRUMENT { - // While we're here, we should patch the nop sled. To do that we mprotect - // the page containing the function to be writeable. - const uint64_t PageSize = GetPageSizeCached(); - void *PageAlignedAddr = - reinterpret_cast<void *>(Sled.Address & ~(PageSize - 1)); - std::size_t MProtectLen = (Sled.Address + cSledLength) - - reinterpret_cast<uint64_t>(PageAlignedAddr); - MProtectHelper Protector(PageAlignedAddr, MProtectLen); - if (Protector.MakeWriteable() == -1) { - printf("Failed mprotect: %d\n", errno); - return XRayPatchingStatus::FAILED; - } +namespace { +bool patchSled(const XRaySledEntry &Sled, bool Enable, + int32_t FuncId) XRAY_NEVER_INSTRUMENT { bool Success = false; switch (Sled.Kind) { case XRayEntryType::ENTRY: @@ -191,6 +123,55 @@ inline bool patchSled(const XRaySledEntry &Sled, bool Enable, return Success; } +XRayPatchingStatus patchFunction(int32_t FuncId, + bool Enable) XRAY_NEVER_INSTRUMENT { + if (!__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) + return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. + + uint8_t NotPatching = false; + if (!__sanitizer::atomic_compare_exchange_strong( + &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel)) + return XRayPatchingStatus::ONGOING; // Already patching. + + // Next, we look for the function index. + XRaySledMap InstrMap; + { + __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); + InstrMap = XRayInstrMap; + } + + // If we don't have an index, we can't patch individual functions. + if (InstrMap.Functions == 0) + return XRayPatchingStatus::NOT_INITIALIZED; + + // FuncId must be a positive number, less than the number of functions + // instrumented. + if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { + Report("Invalid function id provided: %d\n", FuncId); + return XRayPatchingStatus::FAILED; + } + + // Now we patch ths sleds for this specific function. + auto SledRange = InstrMap.SledsIndex[FuncId - 1]; + auto *f = SledRange.Begin; + auto *e = SledRange.End; + + bool SucceedOnce = false; + while (f != e) + SucceedOnce |= patchSled(*f++, Enable, FuncId); + + __sanitizer::atomic_store(&XRayPatching, false, + __sanitizer::memory_order_release); + + if (!SucceedOnce) { + Report("Failed patching any sled for function '%d'.", FuncId); + return XRayPatchingStatus::FAILED; + } + + return XRayPatchingStatus::SUCCESS; +} + // controlPatching implements the common internals of the patching/unpatching // implementation. |Enable| defines whether we're enabling or disabling the // runtime XRay instrumentation. @@ -205,14 +186,13 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { return XRayPatchingStatus::ONGOING; // Already patching. uint8_t PatchingSuccess = false; - auto XRayPatchingStatusResetter = scopeCleanup([&PatchingSuccess] { - if (!PatchingSuccess) - __sanitizer::atomic_store(&XRayPatching, false, - __sanitizer::memory_order_release); - }); + auto XRayPatchingStatusResetter = + __sanitizer::at_scope_exit([&PatchingSuccess] { + if (!PatchingSuccess) + __sanitizer::atomic_store(&XRayPatching, false, + __sanitizer::memory_order_release); + }); - // Step 1: Compute the function id, as a unique identifier per function in the - // instrumentation map. XRaySledMap InstrMap; { __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); @@ -221,16 +201,47 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { if (InstrMap.Entries == 0) return XRayPatchingStatus::NOT_INITIALIZED; - const uint64_t PageSize = GetPageSizeCached(); + uint32_t FuncId = 1; + uint64_t CurFun = 0; + + // First we want to find the bounds for which we have instrumentation points, + // and try to get as few calls to mprotect(...) as possible. We're assuming + // that all the sleds for the instrumentation map are contiguous as a single + // set of pages. When we do support dynamic shared object instrumentation, + // we'll need to do this for each set of page load offsets per DSO loaded. For + // now we're assuming we can mprotect the whole section of text between the + // minimum sled address and the maximum sled address (+ the largest sled + // size). + auto MinSled = InstrMap.Sleds[0]; + auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1]; + for (std::size_t I = 0; I < InstrMap.Entries; I++) { + const auto &Sled = InstrMap.Sleds[I]; + if (Sled.Address < MinSled.Address) + MinSled = Sled; + if (Sled.Address > MaxSled.Address) + MaxSled = Sled; + } + + const size_t PageSize = flags()->xray_page_size_override > 0 + ? flags()->xray_page_size_override + : GetPageSizeCached(); if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { Report("System page size is not a power of two: %lld\n", PageSize); return XRayPatchingStatus::FAILED; } - uint32_t FuncId = 1; - uint64_t CurFun = 0; - for (std::size_t I = 0; I < InstrMap.Entries; I++) { - auto Sled = InstrMap.Sleds[I]; + void *PageAlignedAddr = + reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); + size_t MProtectLen = + (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; + MProtectHelper Protector(PageAlignedAddr, MProtectLen); + if (Protector.MakeWriteable() == -1) { + Report("Failed mprotect: %d\n", errno); + return XRayPatchingStatus::FAILED; + } + + for (std::size_t I = 0; I < InstrMap.Entries; ++I) { + auto &Sled = InstrMap.Sleds[I]; auto F = Sled.Function; if (CurFun == 0) CurFun = F; @@ -246,36 +257,14 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { return XRayPatchingStatus::SUCCESS; } -XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT { - return controlPatching(true); -} - -XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT { - return controlPatching(false); -} - -XRayPatchingStatus patchFunction(int32_t FuncId, - bool Enable) XRAY_NEVER_INSTRUMENT { - if (!__sanitizer::atomic_load(&XRayInitialized, - __sanitizer::memory_order_acquire)) - return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. - - uint8_t NotPatching = false; - if (!__sanitizer::atomic_compare_exchange_strong( - &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel)) - return XRayPatchingStatus::ONGOING; // Already patching. - - // Next, we look for the function index. +XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, + bool Enable) XRAY_NEVER_INSTRUMENT { XRaySledMap InstrMap; { __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); InstrMap = XRayInstrMap; } - // If we don't have an index, we can't patch individual functions. - if (InstrMap.Functions == 0) - return XRayPatchingStatus::NOT_INITIALIZED; - // FuncId must be a positive number, less than the number of functions // instrumented. if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { @@ -283,33 +272,98 @@ XRayPatchingStatus patchFunction(int32_t FuncId, return XRayPatchingStatus::FAILED; } - // Now we patch ths sleds for this specific function. + const size_t PageSize = flags()->xray_page_size_override > 0 + ? flags()->xray_page_size_override + : GetPageSizeCached(); + if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { + Report("Provided page size is not a power of two: %lld\n", PageSize); + return XRayPatchingStatus::FAILED; + } + + // Here we compute the minumum sled and maximum sled associated with a + // particular function ID. auto SledRange = InstrMap.SledsIndex[FuncId - 1]; auto *f = SledRange.Begin; auto *e = SledRange.End; + auto MinSled = *f; + auto MaxSled = *(SledRange.End - 1); + while (f != e) { + if (f->Address < MinSled.Address) + MinSled = *f; + if (f->Address > MaxSled.Address) + MaxSled = *f; + ++f; + } - bool SucceedOnce = false; - while (f != e) - SucceedOnce |= patchSled(*f++, Enable, FuncId); + void *PageAlignedAddr = + reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); + size_t MProtectLen = + (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; + MProtectHelper Protector(PageAlignedAddr, MProtectLen); + if (Protector.MakeWriteable() == -1) { + Report("Failed mprotect: %d\n", errno); + return XRayPatchingStatus::FAILED; + } + return patchFunction(FuncId, Enable); +} - __sanitizer::atomic_store(&XRayPatching, false, - __sanitizer::memory_order_release); +} // namespace - if (!SucceedOnce) { - Report("Failed patching any sled for function '%d'.", FuncId); - return XRayPatchingStatus::FAILED; +} // namespace __xray + +using namespace __xray; + +// The following functions are declared `extern "C" {...}` in the header, hence +// they're defined in the global namespace. + +int __xray_set_handler(void (*entry)(int32_t, + XRayEntryType)) XRAY_NEVER_INSTRUMENT { + if (__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) { + + __sanitizer::atomic_store(&__xray::XRayPatchedFunction, + reinterpret_cast<uintptr_t>(entry), + __sanitizer::memory_order_release); + return 1; } + return 0; +} - return XRayPatchingStatus::SUCCESS; +int __xray_set_customevent_handler(void (*entry)(void *, size_t)) + XRAY_NEVER_INSTRUMENT { + if (__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) { + __sanitizer::atomic_store(&__xray::XRayPatchedCustomEvent, + reinterpret_cast<uintptr_t>(entry), + __sanitizer::memory_order_release); + return 1; + } + return 0; +} + +int __xray_remove_handler() XRAY_NEVER_INSTRUMENT { + return __xray_set_handler(nullptr); +} + +int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT { + return __xray_set_customevent_handler(nullptr); +} + +XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT { + return controlPatching(true); +} + +XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT { + return controlPatching(false); } XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { - return patchFunction(FuncId, true); + return mprotectAndPatchFunction(FuncId, true); } XRayPatchingStatus __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { - return patchFunction(FuncId, false); + return mprotectAndPatchFunction(FuncId, false); } int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) { @@ -331,7 +385,7 @@ uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT { __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); if (FuncId <= 0 || static_cast<size_t>(FuncId) > XRayInstrMap.Functions) return 0; - return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Address + return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function // On PPC, function entries are always aligned to 16 bytes. The beginning of a // sled might be a local entry, which is always +8 based on the global entry. // Always return the global entry. diff --git a/contrib/compiler-rt/lib/xray/xray_interface_internal.h b/contrib/compiler-rt/lib/xray/xray_interface_internal.h index 4a2784612fcb..5811e2b7300a 100644 --- a/contrib/compiler-rt/lib/xray/xray_interface_internal.h +++ b/contrib/compiler-rt/lib/xray/xray_interface_internal.h @@ -28,13 +28,15 @@ struct XRaySledEntry { uint64_t Function; unsigned char Kind; unsigned char AlwaysInstrument; - unsigned char Padding[14]; // Need 32 bytes + unsigned char Version; + unsigned char Padding[13]; // Need 32 bytes #elif SANITIZER_WORDSIZE == 32 uint32_t Address; uint32_t Function; unsigned char Kind; unsigned char AlwaysInstrument; - unsigned char Padding[6]; // Need 16 bytes + unsigned char Version; + unsigned char Padding[5]; // Need 16 bytes #else #error "Unsupported word size." #endif diff --git a/contrib/compiler-rt/lib/xray/xray_log_interface.cc b/contrib/compiler-rt/lib/xray/xray_log_interface.cc index ee14ae4b1b62..783f004d292a 100644 --- a/contrib/compiler-rt/lib/xray/xray_log_interface.cc +++ b/contrib/compiler-rt/lib/xray/xray_log_interface.cc @@ -12,35 +12,84 @@ //===----------------------------------------------------------------------===// #include "xray/xray_log_interface.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_mutex.h" #include "xray/xray_interface.h" #include "xray_defs.h" -#include <memory> - __sanitizer::SpinMutex XRayImplMutex; -std::unique_ptr<XRayLogImpl> GlobalXRayImpl; +XRayLogImpl CurrentXRayImpl{nullptr, nullptr, nullptr, nullptr}; +XRayLogImpl *GlobalXRayImpl = nullptr; + +// We use a linked list of Mode to XRayLogImpl mappings. This is a linked list +// when it should be a map because we're avoiding having to depend on C++ +// standard library data structures at this level of the implementation. +struct ModeImpl { + ModeImpl *Next; + const char *Mode; + XRayLogImpl Impl; +}; + +ModeImpl SentinelModeImpl{ + nullptr, nullptr, {nullptr, nullptr, nullptr, nullptr}}; +ModeImpl *ModeImpls = &SentinelModeImpl; + +XRayLogRegisterStatus +__xray_log_register_mode(const char *Mode, + XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT { + if (Impl.flush_log == nullptr || Impl.handle_arg0 == nullptr || + Impl.log_finalize == nullptr || Impl.log_init == nullptr) + return XRayLogRegisterStatus::XRAY_INCOMPLETE_IMPL; + + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); + // First, look for whether the mode already has a registered implementation. + for (ModeImpl *it = ModeImpls; it != &SentinelModeImpl; it = it->Next) { + if (!__sanitizer::internal_strcmp(Mode, it->Mode)) + return XRayLogRegisterStatus::XRAY_DUPLICATE_MODE; + } + auto *NewModeImpl = + static_cast<ModeImpl *>(__sanitizer::InternalAlloc(sizeof(ModeImpl))); + NewModeImpl->Next = ModeImpls; + NewModeImpl->Mode = __sanitizer::internal_strdup(Mode); + NewModeImpl->Impl = Impl; + ModeImpls = NewModeImpl; + return XRayLogRegisterStatus::XRAY_REGISTRATION_OK; +} + +XRayLogRegisterStatus +__xray_log_select_mode(const char *Mode) XRAY_NEVER_INSTRUMENT { + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); + for (ModeImpl *it = ModeImpls; it != &SentinelModeImpl; it = it->Next) { + if (!__sanitizer::internal_strcmp(Mode, it->Mode)) { + CurrentXRayImpl = it->Impl; + GlobalXRayImpl = &CurrentXRayImpl; + __xray_set_handler(it->Impl.handle_arg0); + return XRayLogRegisterStatus::XRAY_REGISTRATION_OK; + } + } + return XRayLogRegisterStatus::XRAY_MODE_NOT_FOUND; +} void __xray_set_log_impl(XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT { if (Impl.log_init == nullptr || Impl.log_finalize == nullptr || Impl.handle_arg0 == nullptr || Impl.flush_log == nullptr) { __sanitizer::SpinMutexLock Guard(&XRayImplMutex); - GlobalXRayImpl.reset(); + GlobalXRayImpl = nullptr; __xray_remove_handler(); __xray_remove_handler_arg1(); return; } __sanitizer::SpinMutexLock Guard(&XRayImplMutex); - GlobalXRayImpl.reset(new XRayLogImpl); - *GlobalXRayImpl = Impl; + CurrentXRayImpl = Impl; + GlobalXRayImpl = &CurrentXRayImpl; __xray_set_handler(Impl.handle_arg0); } void __xray_remove_log_impl() XRAY_NEVER_INSTRUMENT { __sanitizer::SpinMutexLock Guard(&XRayImplMutex); - GlobalXRayImpl.reset(); + GlobalXRayImpl = nullptr; __xray_remove_handler(); __xray_remove_handler_arg1(); } diff --git a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S index b59eedc4bb1b..350afd9265fd 100644 --- a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S +++ b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S @@ -14,10 +14,13 @@ //===----------------------------------------------------------------------===// #include "../builtins/assembly.h" +#include "../sanitizer_common/sanitizer_asm.h" + + .macro SAVE_REGISTERS subq $192, %rsp - .cfi_def_cfa_offset 200 + CFI_DEF_CFA_OFFSET(200) // At this point, the stack pointer should be aligned to an 8-byte boundary, // because any call instructions that come after this will add another 8 // bytes and therefore align it to 16-bytes. @@ -57,63 +60,82 @@ movq 8(%rsp), %r8 movq 0(%rsp), %r9 addq $192, %rsp - .cfi_def_cfa_offset 8 + CFI_DEF_CFA_OFFSET(8) +.endm + +.macro ALIGNED_CALL_RAX + // Call the logging handler, after aligning the stack to a 16-byte boundary. + // The approach we're taking here uses additional stack space to stash the + // stack pointer twice before aligning the pointer to 16-bytes. If the stack + // was 8-byte aligned, it will become 16-byte aligned -- when restoring the + // pointer, we can always look -8 bytes from the current position to get + // either of the values we've stashed in the first place. + pushq %rsp + pushq (%rsp) + andq $-0x10, %rsp + callq *%rax + movq 8(%rsp), %rsp .endm .text +#if !defined(__APPLE__) + .section .text +#else + .section __TEXT,__text +#endif .file "xray_trampoline_x86.S" //===----------------------------------------------------------------------===// - .globl __xray_FunctionEntry + .globl ASM_SYMBOL(__xray_FunctionEntry) .align 16, 0x90 - .type __xray_FunctionEntry,@function - -__xray_FunctionEntry: - .cfi_startproc + ASM_TYPE_FUNCTION(__xray_FunctionEntry) +ASM_SYMBOL(__xray_FunctionEntry): + CFI_STARTPROC SAVE_REGISTERS // This load has to be atomic, it's concurrent with __xray_patch(). // On x86/amd64, a simple (type-aligned) MOV instruction is enough. - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax, %rax je .Ltmp0 // The patched function prolog puts its xray_instr_map index into %r10d. movl %r10d, %edi xor %esi,%esi - callq *%rax + ALIGNED_CALL_RAX + .Ltmp0: RESTORE_REGISTERS retq -.Ltmp1: - .size __xray_FunctionEntry, .Ltmp1-__xray_FunctionEntry - .cfi_endproc + ASM_SIZE(__xray_FunctionEntry) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .globl __xray_FunctionExit + .globl ASM_SYMBOL(__xray_FunctionExit) .align 16, 0x90 - .type __xray_FunctionExit,@function -__xray_FunctionExit: - .cfi_startproc + ASM_TYPE_FUNCTION(__xray_FunctionExit) +ASM_SYMBOL(__xray_FunctionExit): + CFI_STARTPROC // Save the important registers first. Since we're assuming that this // function is only jumped into, we only preserve the registers for // returning. subq $56, %rsp - .cfi_def_cfa_offset 64 + CFI_DEF_CFA_OFFSET(64) movq %rbp, 48(%rsp) movupd %xmm0, 32(%rsp) movupd %xmm1, 16(%rsp) movq %rax, 8(%rsp) movq %rdx, 0(%rsp) - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax,%rax je .Ltmp2 movl %r10d, %edi movl $1, %esi - callq *%rax + ALIGNED_CALL_RAX + .Ltmp2: // Restore the important registers. movq 48(%rsp), %rbp @@ -122,111 +144,94 @@ __xray_FunctionExit: movq 8(%rsp), %rax movq 0(%rsp), %rdx addq $56, %rsp - .cfi_def_cfa_offset 8 + CFI_DEF_CFA_OFFSET(8) retq -.Ltmp3: - .size __xray_FunctionExit, .Ltmp3-__xray_FunctionExit - .cfi_endproc + ASM_SIZE(__xray_FunctionExit) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .global __xray_FunctionTailExit + .globl ASM_SYMBOL(__xray_FunctionTailExit) .align 16, 0x90 - .type __xray_FunctionTailExit,@function -__xray_FunctionTailExit: - .cfi_startproc - // Save the important registers as in the entry trampoline, but indicate that - // this is an exit. In the future, we will introduce a new entry type that - // differentiates between a normal exit and a tail exit, but we'd have to do - // this and increment the version number for the header. + ASM_TYPE_FUNCTION(__xray_FunctionTailExit) +ASM_SYMBOL(__xray_FunctionTailExit): + CFI_STARTPROC SAVE_REGISTERS - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax,%rax je .Ltmp4 movl %r10d, %edi - movl $1, %esi - callq *%rax + movl $2, %esi + + ALIGNED_CALL_RAX .Ltmp4: RESTORE_REGISTERS retq -.Ltmp5: - .size __xray_FunctionTailExit, .Ltmp5-__xray_FunctionTailExit - .cfi_endproc + ASM_SIZE(__xray_FunctionTailExit) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .globl __xray_ArgLoggerEntry + .globl ASM_SYMBOL(__xray_ArgLoggerEntry) .align 16, 0x90 - .type __xray_ArgLoggerEntry,@function -__xray_ArgLoggerEntry: - .cfi_startproc + ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry) +ASM_SYMBOL(__xray_ArgLoggerEntry): + CFI_STARTPROC SAVE_REGISTERS // Again, these function pointer loads must be atomic; MOV is fine. - movq _ZN6__xray13XRayArgLoggerE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray13XRayArgLoggerE)(%rip), %rax testq %rax, %rax jne .Larg1entryLog // If [arg1 logging handler] not set, defer to no-arg logging. - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax, %rax je .Larg1entryFail .Larg1entryLog: - // First argument will become the third + // First argument will become the third movq %rdi, %rdx - // XRayEntryType::ENTRY into the second - xorq %rsi, %rsi + // XRayEntryType::LOG_ARGS_ENTRY into the second + mov $0x3, %esi // 32-bit function ID becomes the first movl %r10d, %edi - callq *%rax + ALIGNED_CALL_RAX .Larg1entryFail: RESTORE_REGISTERS retq - -.Larg1entryEnd: - .size __xray_ArgLoggerEntry, .Larg1entryEnd-__xray_ArgLoggerEntry - .cfi_endproc + ASM_SIZE(__xray_ArgLoggerEntry) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .global __xray_CustomEvent + .global ASM_SYMBOL(__xray_CustomEvent) .align 16, 0x90 - .type __xray_CustomEvent,@function -__xray_CustomEvent: - .cfi_startproc - subq $16, %rsp - .cfi_def_cfa_offset 24 - movq %rbp, 8(%rsp) - movq %rax, 0(%rsp) + ASM_TYPE_FUNCTION(__xray_CustomEvent) +ASM_SYMBOL(__xray_CustomEvent): + CFI_STARTPROC + SAVE_REGISTERS // We take two arguments to this trampoline, which should be in rdi and rsi // already. We also make sure that we stash %rax because we use that register // to call the logging handler. - movq _ZN6__xray22XRayPatchedCustomEventE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray22XRayPatchedCustomEventE)(%rip), %rax testq %rax,%rax je .LcustomEventCleanup - // At this point we know that rcx and rdx already has the data, so we just - // call the logging handler. - callq *%rax + ALIGNED_CALL_RAX .LcustomEventCleanup: - movq 0(%rsp), %rax - movq 8(%rsp), %rbp - addq $16, %rsp - .cfi_def_cfa_offset 8 + RESTORE_REGISTERS retq - -.Ltmp8: - .size __xray_CustomEvent, .Ltmp8-__xray_CustomEvent - .cfi_endproc + ASM_SIZE(__xray_CustomEvent) + CFI_ENDPROC NO_EXEC_STACK_DIRECTIVE diff --git a/contrib/compiler-rt/lib/xray/xray_utils.cc b/contrib/compiler-rt/lib/xray/xray_utils.cc index b9a38d1b98eb..cf800d3aeaf8 100644 --- a/contrib/compiler-rt/lib/xray/xray_utils.cc +++ b/contrib/compiler-rt/lib/xray/xray_utils.cc @@ -117,7 +117,8 @@ int getLogFD() XRAY_NEVER_INSTRUMENT { TmpFilename); return -1; } - Report("XRay: Log file in '%s'\n", TmpFilename); + if (__sanitizer::Verbosity()) + Report("XRay: Log file in '%s'\n", TmpFilename); return Fd; } diff --git a/contrib/compiler-rt/lib/xray/xray_x86_64.cc b/contrib/compiler-rt/lib/xray/xray_x86_64.cc index e34806fa1cea..e17f00ac3a62 100644 --- a/contrib/compiler-rt/lib/xray/xray_x86_64.cc +++ b/contrib/compiler-rt/lib/xray/xray_x86_64.cc @@ -76,6 +76,7 @@ static constexpr uint8_t CallOpCode = 0xe8; static constexpr uint16_t MovR10Seq = 0xba41; static constexpr uint16_t Jmp9Seq = 0x09eb; static constexpr uint16_t Jmp20Seq = 0x14eb; +static constexpr uint16_t Jmp15Seq = 0x0feb; static constexpr uint8_t JmpOpCode = 0xe9; static constexpr uint8_t RetOpCode = 0xc3; static constexpr uint16_t NopwSeq = 0x9066; @@ -207,8 +208,10 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { // Here we do the dance of replacing the following sled: // + // In Version 0: + // // xray_sled_n: - // jmp +19 // 2 bytes + // jmp +20 // 2 bytes // ... // // With the following: @@ -216,24 +219,35 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId, // nopw // 2 bytes* // ... // - // We need to do this in the following order: // - // 1. Overwrite the 5-byte nop with the call (relative), where (relative) is - // the relative offset to the __xray_CustomEvent trampoline. - // 2. Do a two-byte atomic write over the 'jmp +24' to turn it into a 'nopw'. - // This allows us to "enable" this code once the changes have committed. + // The "unpatch" should just turn the 'nopw' back to a 'jmp +20'. + // + // --- // - // The "unpatch" should just turn the 'nopw' back to a 'jmp +24'. + // In Version 1: + // + // The jump offset is now 15 bytes (0x0f), so when restoring the nopw back + // to a jmp, use 15 bytes instead. // if (Enable) { std::atomic_store_explicit( reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), NopwSeq, std::memory_order_release); } else { - std::atomic_store_explicit( - reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp20Seq, - std::memory_order_release); - } + switch (Sled.Version) { + case 1: + std::atomic_store_explicit( + reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp15Seq, + std::memory_order_release); + break; + case 0: + default: + std::atomic_store_explicit( + reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp20Seq, + std::memory_order_release); + break; + } + } return false; } @@ -244,9 +258,9 @@ bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { // We check whether rdtscp support is enabled. According to the x86_64 manual, // level should be set at 0x80000001, and we should have a look at bit 27 in - // EDX. That's 0x8000000 (or 1u << 26). + // EDX. That's 0x8000000 (or 1u << 27). __get_cpuid(0x80000001, &EAX, &EBX, &ECX, &EDX); - if (!(EDX & (1u << 26))) { + if (!(EDX & (1u << 27))) { Report("Missing rdtscp support.\n"); return false; } |
