diff options
Diffstat (limited to 'lib/tsan/rtl')
63 files changed, 854 insertions, 643 deletions
diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc index ef984a45cd9d..685ca5518009 100644 --- a/lib/tsan/rtl/tsan_clock.cc +++ b/lib/tsan/rtl/tsan_clock.cc @@ -1,9 +1,8 @@ //===-- tsan_clock.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h index a891d7bbd889..6a1d15a2a16d 100644 --- a/lib/tsan/rtl/tsan_clock.h +++ b/lib/tsan/rtl/tsan_clock.h @@ -1,9 +1,8 @@ //===-- tsan_clock.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_debugging.cc b/lib/tsan/rtl/tsan_debugging.cc index 067aeef97c0e..8579db12b666 100644 --- a/lib/tsan/rtl/tsan_debugging.cc +++ b/lib/tsan/rtl/tsan_debugging.cc @@ -1,9 +1,8 @@ //===-- tsan_debugging.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h index 3c775debfb09..293d7deccc31 100644 --- a/lib/tsan/rtl/tsan_defs.h +++ b/lib/tsan/rtl/tsan_defs.h @@ -1,9 +1,8 @@ //===-- tsan_defs.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h index 16dbdf391085..64fc50e95c25 100644 --- a/lib/tsan/rtl/tsan_dense_alloc.h +++ b/lib/tsan/rtl/tsan_dense_alloc.h @@ -1,9 +1,8 @@ //===-- tsan_dense_alloc.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dispatch_defs.h b/lib/tsan/rtl/tsan_dispatch_defs.h new file mode 100644 index 000000000000..6f1d1f75f600 --- /dev/null +++ b/lib/tsan/rtl/tsan_dispatch_defs.h @@ -0,0 +1,66 @@ +//===-- tsan_dispatch_defs.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_DISPATCH_DEFS_H +#define TSAN_DISPATCH_DEFS_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +typedef struct dispatch_object_s {} *dispatch_object_t; + +#define DISPATCH_DECL(name) \ + typedef struct name##_s : public dispatch_object_s {} *name##_t + +DISPATCH_DECL(dispatch_queue); +DISPATCH_DECL(dispatch_source); +DISPATCH_DECL(dispatch_group); +DISPATCH_DECL(dispatch_data); +DISPATCH_DECL(dispatch_semaphore); +DISPATCH_DECL(dispatch_io); + +typedef void (*dispatch_function_t)(void *arg); +typedef void (^dispatch_block_t)(void); +typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data, + int error); + +typedef long dispatch_once_t; // NOLINT +typedef __sanitizer::u64 dispatch_time_t; +typedef int dispatch_fd_t; // NOLINT +typedef unsigned long dispatch_io_type_t; // NOLINT +typedef unsigned long dispatch_io_close_flags_t; // NOLINT + +extern "C" { +void *dispatch_get_context(dispatch_object_t object); +void dispatch_retain(dispatch_object_t object); +void dispatch_release(dispatch_object_t object); + +extern const dispatch_block_t _dispatch_data_destructor_free; +extern const dispatch_block_t _dispatch_data_destructor_munmap; +} // extern "C" + +#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr +#define DISPATCH_DATA_DESTRUCTOR_FREE _dispatch_data_destructor_free +#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap + +#if __has_attribute(noescape) + #define DISPATCH_NOESCAPE __attribute__((__noescape__)) +#else + #define DISPATCH_NOESCAPE +#endif + +// Data types used in dispatch APIs +typedef unsigned long size_t; // NOLINT +typedef unsigned long uintptr_t; // NOLINT +typedef __sanitizer::s64 off_t; +typedef __sanitizer::u16 mode_t; +typedef long long_t; // NOLINT + +#endif // TSAN_DISPATCH_DEFS_H diff --git a/lib/tsan/rtl/tsan_external.cc b/lib/tsan/rtl/tsan_external.cc index 6c0e9477e41e..ba8bb71be43b 100644 --- a/lib/tsan/rtl/tsan_external.cc +++ b/lib/tsan/rtl/tsan_external.cc @@ -1,9 +1,8 @@ //===-- tsan_external.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.cc b/lib/tsan/rtl/tsan_fd.cc index f13a7432ec95..5b562ae68d5a 100644 --- a/lib/tsan/rtl/tsan_fd.cc +++ b/lib/tsan/rtl/tsan_fd.cc @@ -1,9 +1,8 @@ //===-- tsan_fd.cc --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.h b/lib/tsan/rtl/tsan_fd.h index 64dc84f6edd6..ce4f2f73bac6 100644 --- a/lib/tsan/rtl/tsan_fd.h +++ b/lib/tsan/rtl/tsan_fd.h @@ -1,9 +1,8 @@ //===-- tsan_fd.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.cc b/lib/tsan/rtl/tsan_flags.cc index 877fc8b819fd..a5abb183a753 100644 --- a/lib/tsan/rtl/tsan_flags.cc +++ b/lib/tsan/rtl/tsan_flags.cc @@ -1,9 +1,8 @@ //===-- tsan_flags.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -50,7 +49,7 @@ void RegisterTsanFlags(FlagParser *parser, Flags *f) { &f->second_deadlock_stack); } -void InitializeFlags(Flags *f, const char *env) { +void InitializeFlags(Flags *f, const char *env, const char *env_option_name) { SetCommonFlagsDefaults(); { // Override some common flags defaults. @@ -92,9 +91,9 @@ void InitializeFlags(Flags *f, const char *env) { ubsan_parser.ParseString(ubsan_default_options); #endif // Override from command line. - parser.ParseString(env); + parser.ParseString(env, env_option_name); #if TSAN_CONTAINS_UBSAN - ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); #endif // Sanity check. diff --git a/lib/tsan/rtl/tsan_flags.h b/lib/tsan/rtl/tsan_flags.h index 66740def52fa..da27d5b992bc 100644 --- a/lib/tsan/rtl/tsan_flags.h +++ b/lib/tsan/rtl/tsan_flags.h @@ -1,9 +1,8 @@ //===-- tsan_flags.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,8 @@ struct Flags : DDFlags { void ParseFromString(const char *str); }; -void InitializeFlags(Flags *flags, const char *env); +void InitializeFlags(Flags *flags, const char *env, + const char *env_option_name = nullptr); } // namespace __tsan #endif // TSAN_FLAGS_H diff --git a/lib/tsan/rtl/tsan_flags.inc b/lib/tsan/rtl/tsan_flags.inc index d3b678fdd151..bfb74b696e67 100644 --- a/lib/tsan/rtl/tsan_flags.inc +++ b/lib/tsan/rtl/tsan_flags.inc @@ -1,9 +1,8 @@ //===-- tsan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.cc b/lib/tsan/rtl/tsan_ignoreset.cc index cdb90d229980..b2f657939cce 100644 --- a/lib/tsan/rtl/tsan_ignoreset.cc +++ b/lib/tsan/rtl/tsan_ignoreset.cc @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.h b/lib/tsan/rtl/tsan_ignoreset.h index e747d819c758..3e318bd674d9 100644 --- a/lib/tsan/rtl/tsan_ignoreset.h +++ b/lib/tsan/rtl/tsan_ignoreset.h @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index 9e49dfe5d582..9a184c797985 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -154,7 +153,7 @@ const int SIG_SETMASK = 2; #endif #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ - (!cur_thread()->is_inited) + (cur_thread_init(), !cur_thread()->is_inited) namespace __tsan { struct SignalDesc { @@ -399,7 +398,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. @@ -409,7 +408,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); @@ -455,7 +454,7 @@ static void on_exit_wrapper(int status, void *arg) { } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); @@ -487,7 +486,7 @@ static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { } } -static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { +static void SetJmp(ThreadState *thr, uptr sp) { if (!thr->is_inited) // called from libc guts during bootstrap return; // Cleanup old bufs. @@ -495,7 +494,6 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { // Remember the buf. JmpBuf *buf = thr->jmp_bufs.PushBack(); buf->sp = sp; - buf->mangled_sp = mangled_sp; buf->shadow_stack_pos = thr->shadow_stack_pos; ThreadSignalContext *sctx = SigCtx(thr); buf->int_signal_send = sctx ? sctx->int_signal_send : 0; @@ -507,32 +505,11 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { } static void LongJmp(ThreadState *thr, uptr *env) { -#ifdef __powerpc__ - uptr mangled_sp = env[0]; -#elif SANITIZER_FREEBSD - uptr mangled_sp = env[2]; -#elif SANITIZER_NETBSD - uptr mangled_sp = env[6]; -#elif SANITIZER_MAC -# ifdef __aarch64__ - uptr mangled_sp = - (GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? env[12] : env[13]; -# else - uptr mangled_sp = env[2]; -# endif -#elif SANITIZER_LINUX -# ifdef __aarch64__ - uptr mangled_sp = env[13]; -# elif defined(__mips64) - uptr mangled_sp = env[1]; -# else - uptr mangled_sp = env[6]; -# endif -#endif - // Find the saved buf by mangled_sp. + uptr sp = ExtractLongJmpSp(env); + // Find the saved buf with matching sp. for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; - if (buf->mangled_sp == mangled_sp) { + if (buf->sp == sp) { CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); // Unwind the stack. while (thr->shadow_stack_pos > buf->shadow_stack_pos) @@ -554,8 +531,9 @@ static void LongJmp(ThreadState *thr, uptr *env) { } // FIXME: put everything below into a common extern "C" block? -extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { - SetJmp(cur_thread(), sp, mangled_sp); +extern "C" void __tsan_setjmp(uptr sp) { + cur_thread_init(); + SetJmp(cur_thread(), sp); } #if SANITIZER_MAC @@ -665,7 +643,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(size); void *p = 0; { @@ -682,7 +660,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalCalloc(size, n); void *p = 0; { @@ -694,7 +672,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalRealloc(p, size); if (p) invoke_free_hook(p); @@ -706,10 +684,23 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { return p; } +TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { + if (in_symbolizer()) + return InternalReallocArray(p, size, n); + if (p) + invoke_free_hook(p); + { + SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); + p = user_reallocarray(thr, pc, p, size, n); + } + invoke_malloc_hook(p, size); + return p; +} + TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); @@ -719,7 +710,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); @@ -808,14 +799,14 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, align); SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); return user_aligned_alloc(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, GetPageSizeCached()); SCOPED_INTERCEPTOR_RAW(valloc, sz); return user_valloc(thr, pc, sz); @@ -824,7 +815,7 @@ TSAN_INTERCEPTOR(void*, valloc, uptr sz) { #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { uptr PageSize = GetPageSizeCached(); sz = sz ? RoundUpTo(sz, PageSize) : PageSize; return InternalAlloc(sz, nullptr, PageSize); @@ -839,7 +830,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { void *p = InternalAlloc(sz, nullptr, align); if (!p) return errno_ENOMEM; @@ -943,6 +934,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { void *param = p->param; int tid = 0; { + cur_thread_init(); ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; @@ -959,7 +951,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { internal_sched_yield(); Processor *proc = ProcCreate(); ProcWire(proc, thr); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); atomic_store(&p->tid, 0, memory_order_release); } void *res = callback(param); @@ -1051,6 +1043,16 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { return res; } +TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { + { + SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); +#if !SANITIZER_MAC && !SANITIZER_ANDROID + CHECK_EQ(thr, &cur_thread_placeholder); +#endif + } + REAL(pthread_exit)(retval); +} + #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); @@ -1975,6 +1977,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, __sanitizer_siginfo *info, void *ctx) { + cur_thread_init(); ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { @@ -2091,7 +2094,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, } TSAN_INTERCEPTOR(int, fork, int fake) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); @@ -2204,23 +2207,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO -// There interceptors do not seem to be strictly necessary for tsan. -// But we see cases where the interceptors consume 70% of execution time. -// Memory blocks passed to fgetgrent_r are "written to" by tsan several times. -// First, there is some recursion (getgrnam_r calls fgetgrent_r), and each -// function "writes to" the buffer. Then, the same memory is "written to" -// twice, first as buf and then as pwbufp (both of them refer to the same -// addresses). -#undef SANITIZER_INTERCEPT_GETPWENT -#undef SANITIZER_INTERCEPT_GETPWENT_R -#undef SANITIZER_INTERCEPT_FGETPWENT -#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS -#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS // We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR +#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ @@ -2620,6 +2612,9 @@ static void unreachable() { } #endif +// Define default implementation since interception of libdispatch is optional. +SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} + void InitializeInterceptors() { #if !SANITIZER_MAC // We need to setup it early, because functions like dlsym() can call it. @@ -2637,18 +2632,18 @@ void InitializeInterceptors() { InitializeCommonInterceptors(); InitializeSignalInterceptors(); + InitializeLibdispatchInterceptors(); #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. - using __interception::GetRealFunctionAddress; - GetRealFunctionAddress(TSAN_STRING_SETJMP, - (uptr*)&REAL(setjmp_symname), 0, 0); - GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); - GetRealFunctionAddress(TSAN_STRING_SIGSETJMP, - (uptr*)&REAL(sigsetjmp_symname), 0, 0); + using __interception::InterceptFunction; + InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); + InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); + InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, + 0); #if !SANITIZER_NETBSD - GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); + InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif #endif @@ -2662,6 +2657,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(__libc_memalign); TSAN_INTERCEPT(calloc); TSAN_INTERCEPT(realloc); + TSAN_INTERCEPT(reallocarray); TSAN_INTERCEPT(free); TSAN_INTERCEPT(cfree); TSAN_INTERCEPT(munmap); @@ -2677,6 +2673,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + TSAN_INTERCEPT(pthread_exit); #if SANITIZER_LINUX TSAN_INTERCEPT(pthread_tryjoin_np); TSAN_INTERCEPT(pthread_timedjoin_np); diff --git a/lib/tsan/rtl/tsan_interceptors.h b/lib/tsan/rtl/tsan_interceptors.h index 763b46b88c8f..88d1edd775d3 100644 --- a/lib/tsan/rtl/tsan_interceptors.h +++ b/lib/tsan/rtl/tsan_interceptors.h @@ -21,9 +21,17 @@ class ScopedInterceptor { LibIgnore *libignore(); +#if !SANITIZER_GO +INLINE bool in_symbolizer() { + cur_thread_init(); + return UNLIKELY(cur_thread()->in_symbolizer); +} +#endif + } // namespace __tsan #define SCOPED_INTERCEPTOR_RAW(func, ...) \ + cur_thread_init(); \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc index 579c4d0c0b81..99c6df9dfde7 100644 --- a/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors_mac.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,8 +20,10 @@ #include "tsan_interface_ann.h" #include "sanitizer_common/sanitizer_addrhashmap.h" +#include <errno.h> #include <libkern/OSAtomic.h> #include <objc/objc-sync.h> +#include <sys/ucontext.h> #if defined(__has_include) && __has_include(<xpc/xpc.h>) #include <xpc/xpc.h> @@ -30,6 +31,11 @@ typedef long long_t; // NOLINT +extern "C" { +int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); +int setcontext(const ucontext_t *ucp); +} + namespace __tsan { // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, @@ -354,6 +360,31 @@ TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { return result; } +TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { + { + SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); + } + // Bacause of swapcontext() semantics we have no option but to copy its + // impementation here + if (!oucp || !ucp) { + errno = EINVAL; + return -1; + } + ThreadState *thr = cur_thread(); + const int UCF_SWAPPED = 0x80000000; + oucp->uc_onstack &= ~UCF_SWAPPED; + thr->ignore_interceptors++; + int ret = getcontext(oucp); + if (!(oucp->uc_onstack & UCF_SWAPPED)) { + thr->ignore_interceptors--; + if (!ret) { + oucp->uc_onstack |= UCF_SWAPPED; + ret = setcontext(ucp); + } + } + return ret; +} + // On macOS, libc++ is always linked dynamically, so intercepting works the // usual way. #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR diff --git a/lib/tsan/rtl/tsan_interface.cc b/lib/tsan/rtl/tsan_interface.cc index ad9b1fe9a585..508aadb08f62 100644 --- a/lib/tsan/rtl/tsan_interface.cc +++ b/lib/tsan/rtl/tsan_interface.cc @@ -1,9 +1,8 @@ //===-- tsan_interface.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef u32 uint32_t; typedef u64 uint64_t; void __tsan_init() { + cur_thread_init(); Initialize(cur_thread()); } @@ -124,6 +124,31 @@ void __sanitizer_unaligned_store64(uu64 *addr, u64 v) { __tsan_unaligned_write8(addr); *addr = v; } + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(cur_thread(), CALLERPC, flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber)); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber), flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast<ThreadState *>(fiber), name); +} } // extern "C" void __tsan_acquire(void *addr) { diff --git a/lib/tsan/rtl/tsan_interface.h b/lib/tsan/rtl/tsan_interface.h index 203f6b106a96..fac57809aa24 100644 --- a/lib/tsan/rtl/tsan_interface.h +++ b/lib/tsan/rtl/tsan_interface.h @@ -1,9 +1,8 @@ //===-- tsan_interface.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -200,7 +199,7 @@ __extension__ typedef __int128 a128; #endif // Part of ABI, do not change. -// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup +// https://github.com/llvm/llvm-project/blob/master/libcxx/include/atomic typedef enum { mo_relaxed, mo_consume, diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc index a7922a42e42c..e141ddbb751a 100644 --- a/lib/tsan/rtl/tsan_interface_ann.cc +++ b/lib/tsan/rtl/tsan_interface_ann.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_ann.h b/lib/tsan/rtl/tsan_interface_ann.h index 963bcc55afaa..458d61f53356 100644 --- a/lib/tsan/rtl/tsan_interface_ann.h +++ b/lib/tsan/rtl/tsan_interface_ann.h @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc index d334394f5303..a6b7b0f656d3 100644 --- a/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/lib/tsan/rtl/tsan_interface_atomic.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_atomic.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -475,7 +474,7 @@ static morder convert_morder(morder mo) { #define SCOPED_ATOMIC(func, ...) \ ThreadState *const thr = cur_thread(); \ - if (thr->ignore_sync || thr->ignore_interceptors) { \ + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \ ProcessPendingSignals(thr); \ return NoTsanAtomic##func(__VA_ARGS__); \ } \ diff --git a/lib/tsan/rtl/tsan_interface_inl.h b/lib/tsan/rtl/tsan_interface_inl.h index fff83ee17806..bf4a1658625c 100644 --- a/lib/tsan/rtl/tsan_interface_inl.h +++ b/lib/tsan/rtl/tsan_interface_inl.h @@ -1,9 +1,8 @@ //===-- tsan_interface_inl.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.cc b/lib/tsan/rtl/tsan_interface_java.cc index 75e960e629f9..9f227f09589e 100644 --- a/lib/tsan/rtl/tsan_interface_java.cc +++ b/lib/tsan/rtl/tsan_interface_java.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_java.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.h b/lib/tsan/rtl/tsan_interface_java.h index 0bd49ac3c999..5ad16e959093 100644 --- a/lib/tsan/rtl/tsan_interface_java.h +++ b/lib/tsan/rtl/tsan_interface_java.h @@ -1,9 +1,8 @@ //===-- tsan_interface_java.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_libdispatch_mac.cc b/lib/tsan/rtl/tsan_libdispatch.cc index df22888b3f1c..48ac6a2824a8 100644 --- a/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/lib/tsan/rtl/tsan_libdispatch.cc @@ -1,38 +1,26 @@ -//===-- tsan_libdispatch_mac.cc -------------------------------------------===// +//===-- tsan_libdispatch.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // -// Mac-specific libdispatch (GCD) support. +// Support for intercepting libdispatch (GCD). //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC - #include "sanitizer_common/sanitizer_common.h" #include "interception/interception.h" #include "tsan_interceptors.h" -#include "tsan_platform.h" #include "tsan_rtl.h" -#include <Block.h> -#include <dispatch/dispatch.h> -#include <pthread.h> - -// DISPATCH_NOESCAPE is not defined prior to XCode 8. -#ifndef DISPATCH_NOESCAPE -#define DISPATCH_NOESCAPE -#endif - -typedef long long_t; // NOLINT +#include "BlocksRuntime/Block.h" +#include "tsan_dispatch_defs.h" namespace __tsan { + typedef u16 uint16_t; typedef struct { dispatch_queue_t queue; @@ -42,7 +30,7 @@ typedef struct { bool submitted_synchronously; bool is_barrier_block; uptr non_queue_sync_object; -} tsan_block_context_t; +} block_context_t; // The offsets of different fields of the dispatch_queue_t structure, exported // by libdispatch.dylib. @@ -86,13 +74,11 @@ static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { return tq; } -static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, - dispatch_queue_t queue, - void *orig_context, - dispatch_function_t orig_work) { - tsan_block_context_t *new_context = - (tsan_block_context_t *)user_alloc_internal(thr, pc, - sizeof(tsan_block_context_t)); +static block_context_t *AllocContext(ThreadState *thr, uptr pc, + dispatch_queue_t queue, void *orig_context, + dispatch_function_t orig_work) { + block_context_t *new_context = + (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); new_context->queue = queue; new_context->orig_context = orig_context; new_context->orig_work = orig_work; @@ -111,7 +97,7 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, bool serial_task = context->is_barrier_block || is_queue_serial static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; Acquire(thr, pc, submit_sync); @@ -126,7 +112,7 @@ static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, } static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; if (context->submitted_synchronously) Release(thr, pc, submit_sync); @@ -142,7 +128,7 @@ static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, static void dispatch_callback_wrap(void *param) { SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); - tsan_block_context_t *context = (tsan_block_context_t *)param; + block_context_t *context = (block_context_t *)param; dispatch_sync_pre_execute(thr, pc, context); @@ -166,13 +152,13 @@ static void invoke_and_release_block(void *param) { Block_release(block); } -#define DISPATCH_INTERCEPT_B(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ dispatch_block_t heap_block = Block_copy(block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -185,7 +171,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, block, &invoke_block, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -194,11 +180,11 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } -#define DISPATCH_INTERCEPT_F(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, context, work); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -211,7 +197,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, context, work, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -220,18 +206,21 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } +#define DISPATCH_INTERCEPT(name, barrier) \ + DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ + DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ + DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ + DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) + // We wrap dispatch_async, dispatch_sync and friends where we allocate a new // context, which is used to synchronize (we release the context before // submitting, and the callback acquires it before executing the original // callback). -DISPATCH_INTERCEPT_B(dispatch_async, false) -DISPATCH_INTERCEPT_B(dispatch_barrier_async, true) -DISPATCH_INTERCEPT_F(dispatch_async_f, false) -DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) -DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) -DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) -DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) -DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) +DISPATCH_INTERCEPT(dispatch, false) +DISPATCH_INTERCEPT(dispatch_barrier, true) + +DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, + dispatch_queue_t queue, void *context, dispatch_function_t work) TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, dispatch_queue_t queue, dispatch_block_t block) { @@ -239,7 +228,7 @@ TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); Release(thr, pc, (uptr)new_context); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); @@ -324,9 +313,12 @@ TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, return result; } +// Used, but not intercepted. +extern "C" void dispatch_group_enter(dispatch_group_t group); + TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); - // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. + // Acquired in the group notification callback in dispatch_group_notify[_f]. Release(thr, pc, (uptr)group); REAL(dispatch_group_leave)(group); } @@ -336,10 +328,10 @@ TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); dispatch_retain(group); dispatch_group_enter(group); - __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); + __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); WRAP(dispatch_async)(queue, ^(void) { block_copy(); - _Block_release(block_copy); + Block_release(block_copy); WRAP(dispatch_group_leave)(group); dispatch_release(group); }); @@ -358,6 +350,9 @@ TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, }); } +DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) + TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, dispatch_queue_t q, dispatch_block_t block) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); @@ -377,7 +372,7 @@ TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, block(); }); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); new_context->is_barrier_block = true; Release(thr, pc, (uptr)new_context); @@ -395,7 +390,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, if (handler == nullptr) return REAL(dispatch_source_set_event_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0 }; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -424,7 +419,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, if (handler == nullptr) return REAL(dispatch_source_set_cancel_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -455,7 +450,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, if (handler == nullptr) return REAL(dispatch_source_set_registration_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -484,34 +479,54 @@ TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, DISPATCH_NOESCAPE void (^block)(size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); - void *parent_to_child_sync = nullptr; - uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; - void *child_to_parent_sync = nullptr; - uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; - Release(thr, pc, parent_to_child_sync_uptr); + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { SCOPED_INTERCEPTOR_RAW(dispatch_apply); - Acquire(thr, pc, parent_to_child_sync_uptr); + Acquire(thr, pc, parent_to_child_sync); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); block(iteration); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Release(thr, pc, child_to_parent_sync_uptr); + Release(thr, pc, child_to_parent_sync); }; SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); REAL(dispatch_apply)(iterations, queue, new_block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Acquire(thr, pc, child_to_parent_sync_uptr); + Acquire(thr, pc, child_to_parent_sync); +} + +static void invoke_block_iteration(void *param, size_t iteration) { + auto block = (void (^)(size_t)) param; + block(iteration); } TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, dispatch_queue_t queue, void *context, void (*work)(void *, size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); + + // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch + // implements dispatch_apply in terms of dispatch_apply_f. + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; + + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { + SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); + Acquire(thr, pc, parent_to_child_sync); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); work(context, iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, child_to_parent_sync); }; - WRAP(dispatch_apply)(iterations, queue, new_block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Acquire(thr, pc, child_to_parent_sync); } DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) @@ -531,7 +546,7 @@ TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(destructor); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); uptr submit_sync = (uptr)new_context; Release(thr, pc, submit_sync); @@ -546,7 +561,7 @@ typedef void (^cleanup_handler_t)(int error); TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -563,7 +578,7 @@ TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -580,7 +595,7 @@ TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -599,7 +614,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, dispatch_data_t data, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -617,7 +632,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, dispatch_block_t barrier) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { nullptr, nullptr, &invoke_block, false, false, false, 0}; new_context.non_queue_sync_object = (uptr)channel; new_context.is_barrier_block = true; @@ -637,7 +652,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -662,7 +677,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -687,7 +702,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -722,6 +737,46 @@ TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { return REAL(dispatch_resume)(o); } -} // namespace __tsan +void InitializeLibdispatchInterceptors() { + INTERCEPT_FUNCTION(dispatch_async); + INTERCEPT_FUNCTION(dispatch_async_f); + INTERCEPT_FUNCTION(dispatch_sync); + INTERCEPT_FUNCTION(dispatch_sync_f); + INTERCEPT_FUNCTION(dispatch_barrier_async); + INTERCEPT_FUNCTION(dispatch_barrier_async_f); + INTERCEPT_FUNCTION(dispatch_barrier_sync); + INTERCEPT_FUNCTION(dispatch_barrier_sync_f); + INTERCEPT_FUNCTION(dispatch_after); + INTERCEPT_FUNCTION(dispatch_after_f); + INTERCEPT_FUNCTION(dispatch_once); + INTERCEPT_FUNCTION(dispatch_once_f); + INTERCEPT_FUNCTION(dispatch_semaphore_signal); + INTERCEPT_FUNCTION(dispatch_semaphore_wait); + INTERCEPT_FUNCTION(dispatch_group_wait); + INTERCEPT_FUNCTION(dispatch_group_leave); + INTERCEPT_FUNCTION(dispatch_group_async); + INTERCEPT_FUNCTION(dispatch_group_async_f); + INTERCEPT_FUNCTION(dispatch_group_notify); + INTERCEPT_FUNCTION(dispatch_group_notify_f); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); + INTERCEPT_FUNCTION(dispatch_apply); + INTERCEPT_FUNCTION(dispatch_apply_f); + INTERCEPT_FUNCTION(dispatch_data_create); + INTERCEPT_FUNCTION(dispatch_read); + INTERCEPT_FUNCTION(dispatch_write); + INTERCEPT_FUNCTION(dispatch_io_read); + INTERCEPT_FUNCTION(dispatch_io_write); + INTERCEPT_FUNCTION(dispatch_io_barrier); + INTERCEPT_FUNCTION(dispatch_io_create); + INTERCEPT_FUNCTION(dispatch_io_create_with_path); + INTERCEPT_FUNCTION(dispatch_io_create_with_io); + INTERCEPT_FUNCTION(dispatch_io_close); + INTERCEPT_FUNCTION(dispatch_resume); +} -#endif // SANITIZER_MAC +} // namespace __tsan diff --git a/lib/tsan/rtl/tsan_malloc_mac.cc b/lib/tsan/rtl/tsan_malloc_mac.cc index 3cc30724da85..0b874aecb99c 100644 --- a/lib/tsan/rtl/tsan_malloc_mac.cc +++ b/lib/tsan/rtl/tsan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,19 +28,19 @@ using namespace __tsan; void *p = \ user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) #define COMMON_MALLOC_MALLOC(size) \ - if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ + if (in_symbolizer()) return InternalAlloc(size); \ SCOPED_INTERCEPTOR_RAW(malloc, size); \ void *p = user_alloc(thr, pc, size) #define COMMON_MALLOC_REALLOC(ptr, size) \ - if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \ + if (in_symbolizer()) return InternalRealloc(ptr, size); \ SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ void *p = user_realloc(thr, pc, ptr, size) #define COMMON_MALLOC_CALLOC(count, size) \ - if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \ + if (in_symbolizer()) return InternalCalloc(count, size); \ SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ void *p = user_calloc(thr, pc, size, count) #define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ - if (cur_thread()->in_symbolizer) { \ + if (in_symbolizer()) { \ void *p = InternalAlloc(size, nullptr, alignment); \ if (!p) return errno_ENOMEM; \ *memptr = p; \ @@ -50,12 +49,12 @@ using namespace __tsan; SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, alignment, size); \ int res = user_posix_memalign(thr, pc, memptr, alignment, size); #define COMMON_MALLOC_VALLOC(size) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, GetPageSizeCached()); \ SCOPED_INTERCEPTOR_RAW(valloc, size); \ void *p = user_valloc(thr, pc, size) #define COMMON_MALLOC_FREE(ptr) \ - if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ + if (in_symbolizer()) return InternalFree(ptr); \ SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) #define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); @@ -64,6 +63,8 @@ using namespace __tsan; (void)zone_name; \ Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); #define COMMON_MALLOC_NAMESPACE __tsan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0 #include "sanitizer_common/sanitizer_malloc_mac.inc" diff --git a/lib/tsan/rtl/tsan_md5.cc b/lib/tsan/rtl/tsan_md5.cc index 51279c10d17a..bfe0c17bae75 100644 --- a/lib/tsan/rtl/tsan_md5.cc +++ b/lib/tsan/rtl/tsan_md5.cc @@ -1,9 +1,8 @@ //===-- tsan_md5.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -139,6 +138,14 @@ static const void *body(MD5_CTX *ctx, const void *data, ulong_t size) { return ptr; } +#undef F +#undef G +#undef H +#undef I +#undef STEP +#undef SET +#undef GET + void MD5_Init(MD5_CTX *ctx) { ctx->a = 0x67452301; ctx->b = 0xefcdab89; diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc index b160a9736d5d..f4a95d870cab 100644 --- a/lib/tsan/rtl/tsan_mman.cc +++ b/lib/tsan/rtl/tsan_mman.cc @@ -1,9 +1,8 @@ //===-- tsan_mman.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,6 +201,16 @@ void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { return SetErrnoOnNull(p); } +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) { + if (AllocatorMayReturnNull()) + return SetErrnoOnNull(nullptr); + GET_STACK_TRACE_FATAL(thr, pc); + ReportReallocArrayOverflow(size, n, &stack); + } + return user_realloc(thr, pc, p, size * n); +} + void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); ctx->metamap.AllocBlock(thr, pc, p, sz); diff --git a/lib/tsan/rtl/tsan_mman.h b/lib/tsan/rtl/tsan_mman.h index 6042c5c5d96d..467aabdf2b9d 100644 --- a/lib/tsan/rtl/tsan_mman.h +++ b/lib/tsan/rtl/tsan_mman.h @@ -1,9 +1,8 @@ //===-- tsan_mman.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,6 +34,7 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); void *user_alloc(ThreadState *thr, uptr pc, uptr sz); void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr sz, uptr n); void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz); int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, uptr sz); diff --git a/lib/tsan/rtl/tsan_mutex.cc b/lib/tsan/rtl/tsan_mutex.cc index 22afefce3384..bb7531325aed 100644 --- a/lib/tsan/rtl/tsan_mutex.cc +++ b/lib/tsan/rtl/tsan_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutex.h b/lib/tsan/rtl/tsan_mutex.h index 22ee2f33fcc7..80fdc6ed57bb 100644 --- a/lib/tsan/rtl/tsan_mutex.h +++ b/lib/tsan/rtl/tsan_mutex.h @@ -1,9 +1,8 @@ //===-- tsan_mutex.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.cc b/lib/tsan/rtl/tsan_mutexset.cc index 21587770f687..02e5f9da6080 100644 --- a/lib/tsan/rtl/tsan_mutexset.cc +++ b/lib/tsan/rtl/tsan_mutexset.cc @@ -1,9 +1,8 @@ //===-- tsan_mutexset.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.h b/lib/tsan/rtl/tsan_mutexset.h index 605c21a9c08f..d63881f40290 100644 --- a/lib/tsan/rtl/tsan_mutexset.h +++ b/lib/tsan/rtl/tsan_mutexset.h @@ -1,9 +1,8 @@ //===-- tsan_mutexset.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_new_delete.cc b/lib/tsan/rtl/tsan_new_delete.cc index 4f52d3d71eb6..4cbdf703ad28 100644 --- a/lib/tsan/rtl/tsan_new_delete.cc +++ b/lib/tsan/rtl/tsan_new_delete.cc @@ -1,9 +1,8 @@ //===-- tsan_new_delete.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,7 +29,7 @@ DECLARE_REAL(void, free, void *ptr) // TODO(alekseys): throw std::bad_alloc instead of dying on OOM. #define OPERATOR_NEW_BODY(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size); \ void *p = 0; \ { \ @@ -45,7 +44,7 @@ DECLARE_REAL(void, free, void *ptr) return p; #define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, (uptr)align); \ void *p = 0; \ { \ @@ -115,7 +114,7 @@ void *operator new[](__sanitizer::uptr size, std::align_val_t align, #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalFree(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h index 8303c241809b..0d106c4147c8 100644 --- a/lib/tsan/rtl/tsan_platform.h +++ b/lib/tsan/rtl/tsan_platform.h @@ -1,9 +1,8 @@ //===-- tsan_platform.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1012,6 +1011,7 @@ void FlushShadowMemory(); void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive); int ExtractResolvFDs(void *state, int *fds, int nfd); int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); +uptr ExtractLongJmpSp(uptr *env); void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size); int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc index d2ce60709067..ec8606f65d5c 100644 --- a/lib/tsan/rtl/tsan_platform_linux.cc +++ b/lib/tsan/rtl/tsan_platform_linux.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_linux.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -68,12 +67,25 @@ extern "C" void *__libc_stack_end; void *__libc_stack_end = 0; #endif -#if SANITIZER_LINUX && defined(__aarch64__) -void InitializeGuardPtr() __attribute__((visibility("hidden"))); +#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO +# define INIT_LONGJMP_XOR_KEY 1 +#else +# define INIT_LONGJMP_XOR_KEY 0 +#endif + +#if INIT_LONGJMP_XOR_KEY +#include "interception/interception.h" +// Must be declared outside of other namespaces. +DECLARE_REAL(int, _setjmp, void *env) #endif namespace __tsan { +#if INIT_LONGJMP_XOR_KEY +static void InitializeLongjmpXorKey(); +static uptr longjmp_xor_key; +#endif + #ifdef TSAN_RUNTIME_VMA // Runtime detected VMA size. uptr vmaSize; @@ -249,7 +261,8 @@ void InitializePlatform() { // Go maps shadow memory lazily and works fine with limited address space. // Unlimited stack is not a problem as well, because the executable // is not compiled with -pie. - if (!SANITIZER_GO) { +#if !SANITIZER_GO + { bool reexec = false; // TSan doesn't play well with unlimited stack size (as stack // overlaps with shadow memory). If we detect unlimited stack size, @@ -284,17 +297,16 @@ void InitializePlatform() { CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); reexec = true; } - // Initialize the guard pointer used in {sig}{set,long}jump. - InitializeGuardPtr(); + // Initialize the xor key used in {sig}{set,long}jump. + InitializeLongjmpXorKey(); #endif if (reexec) ReExec(); } -#if !SANITIZER_GO CheckAndProtect(); InitTlsSize(); -#endif +#endif // !SANITIZER_GO } #if !SANITIZER_GO @@ -335,6 +347,83 @@ int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { return res; } +// Reverse operation of libc stack pointer mangling +static uptr UnmangleLongJmpSp(uptr mangled_sp) { +#if defined(__x86_64__) +# if SANITIZER_LINUX + // Reverse of: + // xor %fs:0x30, %rsi + // rol $0x11, %rsi + uptr sp; + asm("ror $0x11, %0 \n" + "xor %%fs:0x30, %0 \n" + : "=r" (sp) + : "0" (mangled_sp)); + return sp; +# else + return mangled_sp; +# endif +#elif defined(__aarch64__) +# if SANITIZER_LINUX + return mangled_sp ^ longjmp_xor_key; +# else + return mangled_sp; +# endif +#elif defined(__powerpc64__) + // Reverse of: + // ld r4, -28696(r13) + // xor r4, r3, r4 + uptr xor_key; + asm("ld %0, -28696(%%r13)" : "=r" (xor_key)); + return mangled_sp ^ xor_key; +#elif defined(__mips__) + return mangled_sp; +#else + #error "Unknown platform" +#endif +} + +#ifdef __powerpc__ +# define LONG_JMP_SP_ENV_SLOT 0 +#elif SANITIZER_FREEBSD +# define LONG_JMP_SP_ENV_SLOT 2 +#elif SANITIZER_NETBSD +# define LONG_JMP_SP_ENV_SLOT 6 +#elif SANITIZER_LINUX +# ifdef __aarch64__ +# define LONG_JMP_SP_ENV_SLOT 13 +# elif defined(__mips64) +# define LONG_JMP_SP_ENV_SLOT 1 +# else +# define LONG_JMP_SP_ENV_SLOT 6 +# endif +#endif + +uptr ExtractLongJmpSp(uptr *env) { + uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; + return UnmangleLongJmpSp(mangled_sp); +} + +#if INIT_LONGJMP_XOR_KEY +// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp +// functions) by XORing them with a random key. For AArch64 it is a global +// variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by +// issuing a setjmp and XORing the SP pointer values to derive the key. +static void InitializeLongjmpXorKey() { + // 1. Call REAL(setjmp), which stores the mangled SP in env. + jmp_buf env; + REAL(_setjmp)(env); + + // 2. Retrieve vanilla/mangled SP. + uptr sp; + asm("mov %0, sp" : "=r" (sp)); + uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT]; + + // 3. xor SPs to obtain key. + longjmp_xor_key = mangled_sp ^ sp; +} +#endif + void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { // Check that the thr object is in tls; const uptr thr_beg = (uptr)thr; @@ -362,7 +451,7 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, pthread_cleanup_pop(0); return res; } -#endif +#endif // !SANITIZER_GO #if !SANITIZER_GO void ReplaceSystemMalloc() { } @@ -402,6 +491,10 @@ ThreadState *cur_thread() { return thr; } +void set_cur_thread(ThreadState *thr) { + *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); +} + void cur_thread_finalize() { __sanitizer_sigset_t emptyset; internal_sigfillset(&emptyset); diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc index 7e3a47387b6d..0c2d2aa9338e 100644 --- a/lib/tsan/rtl/tsan_platform_mac.cc +++ b/lib/tsan/rtl/tsan_platform_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_mac.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -74,22 +73,22 @@ static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { // shadow memory is set up. static uptr main_thread_identity = 0; ALIGNED(64) static char main_thread_state[sizeof(ThreadState)]; +static ThreadState *main_thread_state_loc = (ThreadState *)main_thread_state; -ThreadState **cur_thread_location() { - ThreadState **thread_identity = (ThreadState **)pthread_self(); - return ((uptr)thread_identity == main_thread_identity) ? nullptr - : thread_identity; +static ThreadState **cur_thread_location() { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity || main_thread_identity == 0) + return &main_thread_state_loc; + return (ThreadState **)MemToShadow(thread_identity); } ThreadState *cur_thread() { - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr || main_thread_identity == 0) { - return (ThreadState *)&main_thread_state; - } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - ThreadState *thr = (ThreadState *)SignalSafeGetOrAllocate( - (uptr *)fake_tls, sizeof(ThreadState)); - return thr; + return (ThreadState *)SignalSafeGetOrAllocate( + (uptr *)cur_thread_location(), sizeof(ThreadState)); +} + +void set_cur_thread(ThreadState *thr) { + *cur_thread_location() = thr; } // TODO(kuba.brecka): This is not async-signal-safe. In particular, we call @@ -97,14 +96,13 @@ ThreadState *cur_thread() { // handler will try to access the unmapped ThreadState. void cur_thread_finalize() { ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + if (thr_state_loc == &main_thread_state_loc) { // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to // exit the main thread. Let's keep the main thread's ThreadState. return; } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - internal_munmap(*fake_tls, sizeof(ThreadState)); - *fake_tls = nullptr; + internal_munmap(*thr_state_loc, sizeof(ThreadState)); + *thr_state_loc = nullptr; } #endif @@ -213,7 +211,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, ThreadState *parent_thread_state = nullptr; // No parent. int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); CHECK_NE(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ true); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { if (thread == pthread_self()) { @@ -240,8 +238,7 @@ void InitializePlatformEarly() { #endif } -static const uptr kPthreadSetjmpXorKeySlot = 0x7; -extern "C" uptr __tsan_darwin_setjmp_xor_key = 0; +static uptr longjmp_xor_key = 0; void InitializePlatform() { DisableCoreDumperIfNecessary(); @@ -256,21 +253,35 @@ void InitializePlatform() { #endif if (GetMacosVersion() >= MACOS_VERSION_MOJAVE) { - __tsan_darwin_setjmp_xor_key = - (uptr)pthread_getspecific(kPthreadSetjmpXorKeySlot); + // Libsystem currently uses a process-global key; this might change. + const unsigned kTLSLongjmpXorKeySlot = 0x7; + longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot); } } +#ifdef __aarch64__ +# define LONG_JMP_SP_ENV_SLOT \ + ((GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? 12 : 13) +#else +# define LONG_JMP_SP_ENV_SLOT 2 +#endif + +uptr ExtractLongJmpSp(uptr *env) { + uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; + uptr sp = mangled_sp ^ longjmp_xor_key; + return sp; +} + #if !SANITIZER_GO void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { // The pointer to the ThreadState object is stored in the shadow memory // of the tls. uptr tls_end = tls_addr + tls_size; - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity) { MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, tls_size); } else { - uptr thr_state_start = (uptr)thr_state_loc; + uptr thr_state_start = thread_identity; uptr thr_state_end = thr_state_start + sizeof(uptr); CHECK_GE(thr_state_start, tls_addr); CHECK_LE(thr_state_start, tls_addr + tls_size); diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc index c38dcc7f2c1c..3bd0f1bd48d8 100644 --- a/lib/tsan/rtl/tsan_platform_posix.cc +++ b/lib/tsan/rtl/tsan_platform_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_posix.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_platform_windows.cc b/lib/tsan/rtl/tsan_platform_windows.cc index 08aa588a4c89..037297559ee0 100644 --- a/lib/tsan/rtl/tsan_platform_windows.cc +++ b/lib/tsan/rtl/tsan_platform_windows.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_windows.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_preinit.cc b/lib/tsan/rtl/tsan_preinit.cc index a96618d7dc81..052b353091d1 100644 --- a/lib/tsan/rtl/tsan_preinit.cc +++ b/lib/tsan/rtl/tsan_preinit.cc @@ -1,9 +1,8 @@ //===-- tsan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_report.cc b/lib/tsan/rtl/tsan_report.cc index 629c3e9337ab..ae669024a879 100644 --- a/lib/tsan/rtl/tsan_report.cc +++ b/lib/tsan/rtl/tsan_report.cc @@ -1,9 +1,8 @@ //===-- tsan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -258,7 +257,7 @@ static void PrintThread(const ReportThread *rt) { Printf(" '%s'", rt->name); char thrbuf[kThreadBufSize]; const char *thread_status = rt->running ? "running" : "finished"; - if (rt->workerthread) { + if (rt->thread_type == ThreadType::Worker) { Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status); Printf("\n"); Printf("%s", d.Default()); diff --git a/lib/tsan/rtl/tsan_report.h b/lib/tsan/rtl/tsan_report.h index cdc999c6af1d..b4e4d8989379 100644 --- a/lib/tsan/rtl/tsan_report.h +++ b/lib/tsan/rtl/tsan_report.h @@ -1,9 +1,8 @@ //===-- tsan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #define TSAN_REPORT_H #include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_vector.h" #include "tsan_defs.h" @@ -92,7 +92,7 @@ struct ReportThread { int id; tid_t os_id; bool running; - bool workerthread; + ThreadType thread_type; char *name; u32 parent_tid; ReportStack *stack; diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc index f038e9682d89..8a2704ff0631 100644 --- a/lib/tsan/rtl/tsan_rtl.cc +++ b/lib/tsan/rtl/tsan_rtl.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -329,11 +328,8 @@ static void CheckShadowMapping() { #if !SANITIZER_GO static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - uptr top = 0; - uptr bottom = 0; - bool fast = common_flags()->fast_unwind_on_fatal; - if (fast) GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast); + stack->Unwind(sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) { @@ -354,10 +350,11 @@ void Initialize(ThreadState *thr) { SetCheckFailedCallback(TsanCheckFailed); ctx = new(ctx_placeholder) Context; - const char *options = GetEnv(SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS"); + const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS"; + const char *options = GetEnv(env_name); CacheBinaryName(); CheckASLR(); - InitializeFlags(&ctx->flags, options); + InitializeFlags(&ctx->flags, options, env_name); AvoidCVE_2016_2143(); __sanitizer::InitializePlatformEarly(); __tsan::InitializePlatformEarly(); @@ -397,7 +394,7 @@ void Initialize(ThreadState *thr) { // Initialize thread 0. int tid = ThreadCreate(thr, 0, 0, true); CHECK_EQ(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); #if TSAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif @@ -642,6 +639,7 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, // __m128i _mm_move_epi64(__m128i*); // _mm_storel_epi64(u64*, __m128i); u64 store_word = cur.raw(); + bool stored = false; // scan all the shadow values and dispatch to 4 categories: // same, replace, candidate and race (see comments below). @@ -666,16 +664,28 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, int idx = 0; #include "tsan_update_shadow_word_inl.h" idx = 1; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 2; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 3; + if (stored) { #include "tsan_update_shadow_word_inl.h" + } else { +#include "tsan_update_shadow_word_inl.h" + } #endif // we did not find any races and had already stored // the current access info, so we are done - if (LIKELY(store_word == 0)) + if (LIKELY(stored)) return; // choose a random candidate slot and replace it StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); @@ -815,7 +825,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } #endif - if (!SANITIZER_GO && *shadow_mem == kShadowRodata) { + if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. StatInc(thr, StatMop); @@ -826,7 +836,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) { + if (UNLIKELY(fast_state.GetIgnoreBit())) { StatInc(thr, StatMop); StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h index 3410be294502..3a8231bda9a9 100644 --- a/lib/tsan/rtl/tsan_rtl.h +++ b/lib/tsan/rtl/tsan_rtl.h @@ -1,9 +1,8 @@ //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,19 +55,14 @@ namespace __tsan { #if !SANITIZER_GO struct MapUnmapCallback; #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) -static const uptr kAllocatorRegionSizeLog = 20; -static const uptr kAllocatorNumRegions = - SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog; -using ByteMap = TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, - LocalAddressSpaceView, MapUnmapCallback>; + struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = kAllocatorRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __tsan::ByteMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -85,10 +79,8 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; #endif -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; Allocator *allocator(); #endif @@ -333,7 +325,6 @@ struct ThreadSignalContext; struct JmpBuf { uptr sp; - uptr mangled_sp; int int_signal_send; bool in_blocking_func; uptr in_signal_handler; @@ -385,6 +376,9 @@ struct ThreadState { // taken by epoch between synchs. // This way we can save one load from tls. u64 fast_synch_epoch; + // Technically `current` should be a separate THREADLOCAL variable; + // but it is placed here in order to share cache line with previous fields. + ThreadState* current; // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. // We do not distinguish beteween ignoring reads and writes // for better performance. @@ -462,12 +456,22 @@ struct ThreadState { #if !SANITIZER_GO #if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); +void set_cur_thread(ThreadState *thr); void cur_thread_finalize(); +INLINE void cur_thread_init() { } #else __attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; INLINE ThreadState *cur_thread() { - return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); + return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current; +} +INLINE void cur_thread_init() { + ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder); + if (UNLIKELY(!thr->current)) + thr->current = thr; +} +INLINE void set_cur_thread(ThreadState *thr) { + reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID @@ -765,7 +769,8 @@ void FuncEntry(ThreadState *thr, uptr pc); void FuncExit(ThreadState *thr); int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread); +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type); void ThreadFinish(ThreadState *thr); int ThreadTid(ThreadState *thr, uptr pc, uptr uid); void ThreadJoin(ThreadState *thr, uptr pc, int tid); @@ -868,6 +873,16 @@ uptr ALWAYS_INLINE HeapEnd() { } #endif +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); +void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); + +// These need to match __tsan_switch_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum FiberSwitchFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + } // namespace __tsan #endif // TSAN_RTL_H diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S index 3d02bf22f8a5..e0b4c71dfed9 100644 --- a/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -3,14 +3,6 @@ #include "sanitizer_common/sanitizer_asm.h" -#if !defined(__APPLE__) -.section .bss -.type __tsan_pointer_chk_guard, %object -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) -__tsan_pointer_chk_guard: -.zero 8 -#endif - #if defined(__APPLE__) .align 2 @@ -40,57 +32,6 @@ _sigsetjmp$non_lazy_ptr: .align 3 #endif -#if !defined(__APPLE__) -// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp -// functions) by XORing them with a random guard pointer. For AArch64 it is a -// global variable rather than a TCB one (as for x86_64/powerpc) and althought -// its value is exported by the loader, it lies within a private GLIBC -// namespace (meaning it should be only used by GLIBC itself and the ABI is -// not stable). So InitializeGuardPtr obtains the pointer guard value by -// issuing a setjmp and checking the resulting pointers values against the -// original ones. -ASM_HIDDEN(_Z18InitializeGuardPtrv) -.global _Z18InitializeGuardPtrv -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) -_Z18InitializeGuardPtrv: - CFI_STARTPROC - // Allocates a jmp_buf for the setjmp call. - stp x29, x30, [sp, -336]! - CFI_DEF_CFA_OFFSET (336) - CFI_OFFSET (29, -336) - CFI_OFFSET (30, -328) - add x29, sp, 0 - CFI_DEF_CFA_REGISTER (29) - add x0, x29, 24 - - // Call libc setjmp that mangle the stack pointer value - adrp x1, :got:_ZN14__interception12real__setjmpE - ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE] - ldr x1, [x1] - blr x1 - - // glibc setjmp mangles both the frame pointer (FP, pc+4 on blr) and the - // stack pointer (SP). FP will be placed on ((uintptr*)jmp_buf)[11] and - // SP at ((uintptr*)jmp_buf)[13]. - // The mangle operation is just 'value' xor 'pointer guard value' and - // if we know the original value (SP) and the expected one, we can derive - // the guard pointer value. - mov x0, sp - - // Loads the mangled SP pointer. - ldr x1, [x29, 128] - eor x0, x0, x1 - adrp x2, __tsan_pointer_chk_guard - str x0, [x2, #:lo12:__tsan_pointer_chk_guard] - ldp x29, x30, [sp], 336 - CFI_RESTORE (30) - CFI_RESTORE (19) - CFI_DEF_CFA (31, 0) - ret - CFI_ENDPROC -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) -#endif - ASM_HIDDEN(__tsan_setjmp) .comm _ZN14__interception11real_setjmpE,8,8 .globl ASM_SYMBOL_INTERCEPTOR(setjmp) @@ -98,7 +39,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) ASM_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -108,33 +49,24 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf - str x19, [sp, 16] - CFI_OFFSET (19, -16) - mov x19, x0 + // Save env parameter + str x0, [sp, 16] + CFI_OFFSET (0, -16) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] - add x0, x29, 32 - eor x1, x2, x0 -#else - adrp x2, ___tsan_darwin_setjmp_xor_key@page - ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - // restore env parameter - mov x0, x19 - ldr x19, [sp, 16] + // Restore env parameter + ldr x0, [sp, 16] + CFI_RESTORE (0) + + // Restore frame/link register ldp x29, x30, [sp], 32 + CFI_RESTORE (29) CFI_RESTORE (30) - CFI_RESTORE (19) CFI_DEF_CFA (31, 0) // tail jump to libc setjmp @@ -158,7 +90,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) ASM_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -168,33 +100,24 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf - str x19, [sp, 16] - CFI_OFFSET (19, -16) - mov x19, x0 + // Save env parameter + str x0, [sp, 16] + CFI_OFFSET (0, -16) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#else - adrp x2, ___tsan_darwin_setjmp_xor_key@page - ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] - add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - // Restore jmp_buf parameter - mov x0, x19 - ldr x19, [sp, 16] + // Restore env parameter + ldr x0, [sp, 16] + CFI_RESTORE (0) + + // Restore frame/link register ldp x29, x30, [sp], 32 + CFI_RESTORE (29) CFI_RESTORE (30) - CFI_RESTORE (19) CFI_DEF_CFA (31, 0) // tail jump to libc setjmp @@ -218,7 +141,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) ASM_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -228,38 +151,26 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf and savesigs - stp x19, x20, [sp, 16] - CFI_OFFSET (19, -16) - CFI_OFFSET (20, -8) - mov w20, w1 - mov x19, x0 + // Save env and savesigs parameter + stp x0, x1, [sp, 16] + CFI_OFFSET (0, -16) + CFI_OFFSET (1, -8) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] - add x0, x29, 32 - eor x1, x2, x0 -#else - adrp x2, ___tsan_darwin_setjmp_xor_key@page - ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - // restore env parameter - mov w1, w20 - mov x0, x19 - ldp x19, x20, [sp, 16] + // Restore env and savesigs parameter + ldp x0, x1, [sp, 16] + CFI_RESTORE (0) + CFI_RESTORE (1) + + // Restore frame/link register ldp x29, x30, [sp], 32 - CFI_RESTORE (30) CFI_RESTORE (29) - CFI_RESTORE (19) - CFI_RESTORE (20) + CFI_RESTORE (30) CFI_DEF_CFA (31, 0) // tail jump to libc sigsetjmp @@ -283,7 +194,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -293,32 +204,26 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf and savesigs - stp x19, x20, [sp, 16] - CFI_OFFSET (19, -16) - CFI_OFFSET (20, -8) - mov w20, w1 - mov x19, x0 + // Save env and savesigs parameter + stp x0, x1, [sp, 16] + CFI_OFFSET (0, -16) + CFI_OFFSET (1, -8) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - mov w1, w20 - mov x0, x19 - ldp x19, x20, [sp, 16] + // Restore env and savesigs parameter + ldp x0, x1, [sp, 16] + CFI_RESTORE (0) + CFI_RESTORE (1) + + // Restore frame/link register ldp x29, x30, [sp], 32 - CFI_RESTORE (30) CFI_RESTORE (29) - CFI_RESTORE (19) - CFI_RESTORE (20) + CFI_RESTORE (30) CFI_DEF_CFA (31, 0) // tail jump to libc __sigsetjmp @@ -335,9 +240,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif -#if defined(__linux__) -/* We do not need executable stack. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S index 34ef51c2a725..5913aa360c5d 100644 --- a/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/lib/tsan/rtl/tsan_rtl_amd64.S @@ -189,19 +189,11 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi - mov %rdi, %rsi -#elif defined(__APPLE__) +#elif defined(__linux__) || defined(__APPLE__) lea 16(%rsp), %rdi - mov %rdi, %rsi - xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi -#elif defined(__linux__) - lea 16(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #else # error "Unknown platform" #endif @@ -238,19 +230,11 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi - mov %rdi, %rsi -#elif defined(__APPLE__) - lea 16(%rsp), %rdi - mov %rdi, %rsi - xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi -#elif defined(__linux__) +#elif defined(__linux__) || defined(__APPLE__) lea 16(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #else # error "Unknown platform" #endif @@ -294,19 +278,11 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): // align stack frame sub $8, %rsp CFI_ADJUST_CFA_OFFSET(8) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) || defined(__NetBSD__) lea 24(%rsp), %rdi - mov %rdi, %rsi -#elif defined(__APPLE__) +#elif defined(__linux__) || defined(__APPLE__) lea 32(%rsp), %rdi - mov %rdi, %rsi - xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi -#elif defined(__linux__) - lea 32(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #else # error "Unknown platform" #endif @@ -358,15 +334,11 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): // align stack frame sub $8, %rsp CFI_ADJUST_CFA_OFFSET(8) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) lea 24(%rsp), %rdi - mov %rdi, %rsi #else lea 32(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #endif // call tsan interceptor call ASM_SYMBOL(__tsan_setjmp) @@ -389,10 +361,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif // !defined(__APPLE__) && !defined(__NetBSD__) -#if defined(__FreeBSD__) || defined(__linux__) -/* We do not need executable stack. */ -/* This note is not needed on NetBSD. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc index c61d02b7a766..941e70f98872 100644 --- a/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_mutex.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_proc.cc b/lib/tsan/rtl/tsan_rtl_proc.cc index efccdb590ab3..94bbed25b8f5 100644 --- a/lib/tsan/rtl/tsan_rtl_proc.cc +++ b/lib/tsan/rtl/tsan_rtl_proc.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_proc.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc index febb6cef2d39..220a425a2c5b 100644 --- a/lib/tsan/rtl/tsan_rtl_report.cc +++ b/lib/tsan/rtl/tsan_rtl_report.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_report.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,7 +201,7 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { rt->running = (tctx->status == ThreadStatusRunning); rt->name = internal_strdup(tctx->name); rt->parent_tid = tctx->parent_tid; - rt->workerthread = tctx->workerthread; + rt->thread_type = tctx->thread_type; rt->stack = 0; rt->stack = SymbolizeStackId(tctx->creation_stack_id); if (rt->stack) @@ -730,10 +729,12 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) { ALWAYS_INLINE void PrintCurrentStackSlow(uptr pc) { #if !SANITIZER_GO + uptr bp = GET_CURRENT_FRAME(); BufferedStackTrace *ptrace = new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace))) BufferedStackTrace(); - ptrace->Unwind(kStackTraceMax, pc, 0, 0, 0, 0, false); + ptrace->Unwind(pc, bp, nullptr, false); + for (uptr i = 0; i < ptrace->size / 2; i++) { uptr tmp = ptrace->trace_buffer[i]; ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1]; diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc index 766a0f5a505f..fd95cfed4f51 100644 --- a/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/lib/tsan/rtl/tsan_rtl_thread.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_thread.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -240,13 +239,15 @@ int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { return tid; } -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type) { uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (thread_type != ThreadType::Fiber) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) @@ -258,7 +259,7 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { ThreadRegistry *tr = ctx->thread_registry; OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; - tr->StartThread(tid, os_id, workerthread, &args); + tr->StartThread(tid, os_id, thread_type, &args); tr->Lock(); thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); @@ -404,4 +405,40 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, } } +#if !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) { + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast<ThreadState *>(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, ThreadType::Fiber); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) { + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + internal_free(fiber); +} + +void FiberSwitch(ThreadState *thr, uptr pc, + ThreadState *fiber, unsigned flags) { + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + } // namespace __tsan diff --git a/lib/tsan/rtl/tsan_stack_trace.cc b/lib/tsan/rtl/tsan_stack_trace.cc index a0dee19e246e..dbaca23c68aa 100644 --- a/lib/tsan/rtl/tsan_stack_trace.cc +++ b/lib/tsan/rtl/tsan_stack_trace.cc @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,3 +48,16 @@ void VarSizeStackTrace::ReverseOrder() { } } // namespace __tsan + +#if !SANITIZER_GO +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + uptr top = 0; + uptr bottom = 0; + if (StackTrace::WillUseFastUnwind(request_fast)) { + GetThreadStackTopAndBottom(false, &top, &bottom); + Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} +#endif // SANITIZER_GO diff --git a/lib/tsan/rtl/tsan_stack_trace.h b/lib/tsan/rtl/tsan_stack_trace.h index f69b57464bcf..3eb8ce156e83 100644 --- a/lib/tsan/rtl/tsan_stack_trace.h +++ b/lib/tsan/rtl/tsan_stack_trace.h @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc index 18c83d5c6dac..d23ff47d9af0 100644 --- a/lib/tsan/rtl/tsan_stat.cc +++ b/lib/tsan/rtl/tsan_stat.cc @@ -1,9 +1,8 @@ //===-- tsan_stat.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h index 42d6a2b63532..94e18bc66df9 100644 --- a/lib/tsan/rtl/tsan_stat.h +++ b/lib/tsan/rtl/tsan_stat.h @@ -1,9 +1,8 @@ //===-- tsan_stat.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.cc b/lib/tsan/rtl/tsan_suppressions.cc index 6df074118a10..b3eea9ab5869 100644 --- a/lib/tsan/rtl/tsan_suppressions.cc +++ b/lib/tsan/rtl/tsan_suppressions.cc @@ -1,9 +1,8 @@ //===-- tsan_suppressions.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.h b/lib/tsan/rtl/tsan_suppressions.h index 526952d5bd15..f430aeb6c4cf 100644 --- a/lib/tsan/rtl/tsan_suppressions.h +++ b/lib/tsan/rtl/tsan_suppressions.h @@ -1,9 +1,8 @@ //===-- tsan_suppressions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.cc b/lib/tsan/rtl/tsan_symbolize.cc index 27f0e01c7fbe..cb60763f42f8 100644 --- a/lib/tsan/rtl/tsan_symbolize.cc +++ b/lib/tsan/rtl/tsan_symbolize.cc @@ -1,9 +1,8 @@ //===-- tsan_symbolize.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.h b/lib/tsan/rtl/tsan_symbolize.h index 5a9710a3cd30..7adaa04dc273 100644 --- a/lib/tsan/rtl/tsan_symbolize.h +++ b/lib/tsan/rtl/tsan_symbolize.h @@ -1,9 +1,8 @@ //===-- tsan_symbolize.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc index ba3953375ee9..c613b116e3a8 100644 --- a/lib/tsan/rtl/tsan_sync.cc +++ b/lib/tsan/rtl/tsan_sync.cc @@ -1,9 +1,8 @@ //===-- tsan_sync.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.h b/lib/tsan/rtl/tsan_sync.h index 9039970bcf86..47f2739d8de5 100644 --- a/lib/tsan/rtl/tsan_sync.h +++ b/lib/tsan/rtl/tsan_sync.h @@ -1,9 +1,8 @@ //===-- tsan_sync.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_trace.h b/lib/tsan/rtl/tsan_trace.h index 9aef375cbfe6..fbd0f72db6e7 100644 --- a/lib/tsan/rtl/tsan_trace.h +++ b/lib/tsan/rtl/tsan_trace.h @@ -1,9 +1,8 @@ //===-- tsan_trace.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_update_shadow_word_inl.h b/lib/tsan/rtl/tsan_update_shadow_word_inl.h index 6e3ac2fa16f6..056c3aa20320 100644 --- a/lib/tsan/rtl/tsan_update_shadow_word_inl.h +++ b/lib/tsan/rtl/tsan_update_shadow_word_inl.h @@ -1,9 +1,8 @@ //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,31 +17,35 @@ do { const unsigned kAccessSize = 1 << kAccessSizeLog; u64 *sp = &shadow_mem[idx]; old = LoadShadow(sp); - if (old.IsZero()) { + if (LIKELY(old.IsZero())) { StatInc(thr, StatShadowZero); - if (store_word) + if (!stored) { StoreIfNotYetStored(sp, &store_word); - // The above StoreIfNotYetStored could be done unconditionally - // and it even shows 4% gain on synthetic benchmarks (r4307). + stored = true; + } break; } // is the memory access equal to the previous? - if (Shadow::Addr0AndSizeAreEqual(cur, old)) { + if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) { StatInc(thr, StatShadowSameSize); // same thread? - if (Shadow::TidsAreEqual(old, cur)) { + if (LIKELY(Shadow::TidsAreEqual(old, cur))) { StatInc(thr, StatShadowSameThread); - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } StatInc(thr, StatShadowAnotherThread); if (HappensBefore(old, thr)) { - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } - if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))) break; goto RACE; } @@ -56,7 +59,7 @@ do { StatInc(thr, StatShadowAnotherThread); if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) break; - if (HappensBefore(old, thr)) + if (LIKELY(HappensBefore(old, thr))) break; goto RACE; } |