vendor/compiler-rt/compiler-rt-trunk-r301939

29 files changed, 465 insertions, 241 deletions

diff --git a/lib/asan/asan_globals.cc b/lib/asan/asan_globals.cc
index b7233067358c..eebada804f07 100644
--- a/lib/asan/asan_globals.cc
+++ b/lib/asan/asan_globals.cc
@@ -332,6 +332,26 @@ void __asan_unregister_image_globals(uptr *flag) {
   *flag = 0;
 }
 
+void __asan_register_elf_globals(uptr *flag, void *start, void *stop) {
+  if (*flag) return;
+  if (!start) return;
+  CHECK_EQ(0, ((uptr)stop - (uptr)start) % sizeof(__asan_global));
+  __asan_global *globals_start = (__asan_global*)start;
+  __asan_global *globals_stop = (__asan_global*)stop;
+  __asan_register_globals(globals_start, globals_stop - globals_start);
+  *flag = 1;
+}
+
+void __asan_unregister_elf_globals(uptr *flag, void *start, void *stop) {
+  if (!*flag) return;
+  if (!start) return;
+  CHECK_EQ(0, ((uptr)stop - (uptr)start) % sizeof(__asan_global));
+  __asan_global *globals_start = (__asan_global*)start;
+  __asan_global *globals_stop = (__asan_global*)stop;
+  __asan_unregister_globals(globals_start, globals_stop - globals_start);
+  *flag = 0;
+}
+
 // Register an array of globals.
 void __asan_register_globals(__asan_global *globals, uptr n) {
   if (!flags()->report_globals) return;
diff --git a/lib/asan/asan_interceptors.cc b/lib/asan/asan_interceptors.cc
index 6ee3266062f8..905dd2e23870 100644
--- a/lib/asan/asan_interceptors.cc
+++ b/lib/asan/asan_interceptors.cc
@@ -37,12 +37,19 @@
 namespace __asan {
 
 // Return true if we can quickly decide that the region is unpoisoned.
+// We assume that a redzone is at least 16 bytes.
 static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) {
   if (size == 0) return true;
   if (size <= 32)
     return !AddressIsPoisoned(beg) &&
            !AddressIsPoisoned(beg + size - 1) &&
            !AddressIsPoisoned(beg + size / 2);
+  if (size <= 64)
+    return !AddressIsPoisoned(beg) &&
+           !AddressIsPoisoned(beg + size / 4) &&
+           !AddressIsPoisoned(beg + size - 1) &&
+           !AddressIsPoisoned(beg + 3 * size / 4) &&
+           !AddressIsPoisoned(beg + size / 2);
   return false;
 }
 
diff --git a/lib/asan/asan_interface.inc b/lib/asan/asan_interface.inc
index 351be4da5108..e65f61722b10 100644
--- a/lib/asan/asan_interface.inc
+++ b/lib/asan/asan_interface.inc
@@ -64,6 +64,7 @@ INTERFACE_FUNCTION(__asan_poison_stack_memory)
 INTERFACE_FUNCTION(__asan_print_accumulated_stats)
 INTERFACE_FUNCTION(__asan_region_is_poisoned)
 INTERFACE_FUNCTION(__asan_register_globals)
+INTERFACE_FUNCTION(__asan_register_elf_globals)
 INTERFACE_FUNCTION(__asan_register_image_globals)
 INTERFACE_FUNCTION(__asan_report_error)
 INTERFACE_FUNCTION(__asan_report_exp_load1)
@@ -149,6 +150,7 @@ INTERFACE_FUNCTION(__asan_unpoison_intra_object_redzone)
 INTERFACE_FUNCTION(__asan_unpoison_memory_region)
 INTERFACE_FUNCTION(__asan_unpoison_stack_memory)
 INTERFACE_FUNCTION(__asan_unregister_globals)
+INTERFACE_FUNCTION(__asan_unregister_elf_globals)
 INTERFACE_FUNCTION(__asan_unregister_image_globals)
 INTERFACE_FUNCTION(__asan_version_mismatch_check_v8)
 INTERFACE_FUNCTION(__sanitizer_finish_switch_fiber)
diff --git a/lib/asan/asan_interface_internal.h b/lib/asan/asan_interface_internal.h
index b18c31548860..b974c0cc4b43 100644
--- a/lib/asan/asan_interface_internal.h
+++ b/lib/asan/asan_interface_internal.h
@@ -67,6 +67,11 @@ extern "C" {
   SANITIZER_INTERFACE_ATTRIBUTE
   void __asan_unregister_image_globals(uptr *flag);
 
+  SANITIZER_INTERFACE_ATTRIBUTE
+  void __asan_register_elf_globals(uptr *flag, void *start, void *stop);
+  SANITIZER_INTERFACE_ATTRIBUTE
+  void __asan_unregister_elf_globals(uptr *flag, void *start, void *stop);
+
   // These two functions should be called by the instrumented code.
   // 'globals' is an array of structures describing 'n' globals.
   SANITIZER_INTERFACE_ATTRIBUTE
diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
index bb59c344edc2..754ece9840ef 100644
--- a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
@@ -84,7 +84,6 @@ class CoverageData {
   void AfterFork(int child_pid);
   void Extend(uptr npcs);
   void Add(uptr pc, u32 *guard);
-  void DumpAsBitSet();
   void DumpOffsets();
   void DumpAll();
 
@@ -156,6 +155,13 @@ void CoverageData::DirectOpen() {
 
 void CoverageData::Init() {
   pc_fd = kInvalidFd;
+
+  if (!common_flags()->coverage) return;
+  Printf("**\n***\n***\n");
+  Printf("**WARNING: this implementation of SanitizerCoverage is deprecated\n");
+  Printf("**WARNING: and will be removed in future versions\n");
+  Printf("**WARNING: See https://clang.llvm.org/docs/SanitizerCoverage.html\n");
+  Printf("**\n***\n***\n");
 }
 
 void CoverageData::Enable() {
@@ -165,6 +171,8 @@ void CoverageData::Enable() {
       MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit"));
   atomic_store(&pc_array_index, 0, memory_order_relaxed);
   if (common_flags()->coverage_direct) {
+    Report("coverage_direct=1 is deprecated, don't use it.\n");
+    Die();
     atomic_store(&pc_array_size, 0, memory_order_relaxed);
   } else {
     atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed);
@@ -419,35 +427,6 @@ static fd_t CovOpenFile(InternalScopedString *path, bool packed,
   return fd;
 }
 
-void CoverageData::DumpAsBitSet() {
-  if (!common_flags()->coverage_bitset) return;
-  if (!size()) return;
-  InternalScopedBuffer<char> out(size());
-  InternalScopedString path(kMaxPathLength);
-  for (uptr m = 0; m < module_name_vec.size(); m++) {
-    uptr n_set_bits = 0;
-    auto r = module_name_vec[m];
-    CHECK(r.copied_module_name);
-    CHECK_LE(r.beg, r.end);
-    CHECK_LE(r.end, size());
-    for (uptr i = r.beg; i < r.end; i++) {
-      uptr pc = UnbundlePc(pc_array[i]);
-      out[i] = pc ? '1' : '0';
-      if (pc)
-        n_set_bits++;
-    }
-    const char *base_name = StripModuleName(r.copied_module_name);
-    fd_t fd = CovOpenFile(&path, /* packed */false, base_name, "bitset-sancov");
-    if (fd == kInvalidFd) return;
-    WriteToFile(fd, out.data() + r.beg, r.end - r.beg);
-    CloseFile(fd);
-    VReport(1,
-            " CovDump: bitset of %zd bits written for '%s', %zd bits are set\n",
-            r.end - r.beg, base_name, n_set_bits);
-  }
-}
-
-
 void CoverageData::GetRangeOffsets(const NamedPcRange& r, Symbolizer* sym,
     InternalMmapVector<uptr>* offsets) const {
   offsets->clear();
@@ -565,7 +544,6 @@ void CoverageData::DumpAll() {
   if (!coverage_enabled || common_flags()->coverage_direct) return;
   if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
     return;
-  DumpAsBitSet();
   DumpOffsets();
 }
 
diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc
index 73c36082bc67..6d8e3e041cc0 100644
--- a/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc
+++ b/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc
@@ -98,10 +98,6 @@ static void SanitizerDumpCoverage(const uptr* unsorted_pcs, uptr len) {
   InternalFree(file_path);
   InternalFree(module_name);
   InternalFree(pcs);
-
-  if (sancov_flags()->symbolize) {
-    Printf("TODO(aizatsky): call sancov to symbolize\n");
-  }
 }
 
 // Collects trace-pc guard coverage.
diff --git a/lib/sanitizer_common/sanitizer_flags.inc b/lib/sanitizer_common/sanitizer_flags.inc
index 40f8b6204cda..7a5fffcf6165 100644
--- a/lib/sanitizer_common/sanitizer_flags.inc
+++ b/lib/sanitizer_common/sanitizer_flags.inc
@@ -142,12 +142,6 @@ COMMON_FLAG(bool, coverage_pcs, true,
 COMMON_FLAG(bool, coverage_order_pcs, false,
              "If true, the PCs will be dumped in the order they've"
              " appeared during the execution.")
-COMMON_FLAG(bool, coverage_bitset, false,
-            "If set (and if 'coverage' is set too), the coverage information "
-            "will also be dumped as a bitset to a separate file.")
-COMMON_FLAG(bool, coverage_counters, false,
-            "If set (and if 'coverage' is set too), the bitmap that corresponds"
-            " to coverage counters will be dumped.")
 COMMON_FLAG(bool, coverage_direct, SANITIZER_ANDROID,
             "If set, coverage information will be dumped directly to a memory "
             "mapped file. This way data is not lost even if the process is "
diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h
index 14047b4803f8..ee336f7ddff3 100644
--- a/lib/sanitizer_common/sanitizer_linux.h
+++ b/lib/sanitizer_common/sanitizer_linux.h
@@ -88,6 +88,46 @@ bool LibraryNameIs(const char *full_name, const char *base_name);
 
 // Call cb for each region mapped by map.
 void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr));
+
+#if SANITIZER_ANDROID
+
+#if defined(__aarch64__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mrs %0, tpidr_el0" : "=r"(__v)); __v; })
+#elif defined(__arm__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); __v; })
+#elif defined(__mips__)
+// On mips32r1, this goes via a kernel illegal instruction trap that's
+// optimized for v1.
+# define __get_tls() \
+    ({ register void** __v asm("v1"); \
+       __asm__(".set    push\n" \
+               ".set    mips32r2\n" \
+               "rdhwr   %0,$29\n" \
+               ".set    pop\n" : "=r"(__v)); \
+       __v; })
+#elif defined(__i386__)
+# define __get_tls() \
+    ({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })
+#elif defined(__x86_64__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mov %%fs:0, %0" : "=r"(__v)); __v; })
+#else
+#error "Unsupported architecture."
+#endif
+
+// The Android Bionic team has allocated a TLS slot for TSan starting with N,
+// given that Android currently doesn't support ELF TLS. It is used to store
+// Sanitizers thread specific data.
+static const int TLS_SLOT_TSAN = 8;
+
+ALWAYS_INLINE uptr *get_android_tls_ptr() {
+  return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_TSAN]);
+}
+
+#endif  // SANITIZER_ANDROID
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_FREEBSD || SANITIZER_LINUX
diff --git a/lib/scudo/CMakeLists.txt b/lib/scudo/CMakeLists.txt
index ba5e8acd8598..3a8f4ae4fa9e 100644
--- a/lib/scudo/CMakeLists.txt
+++ b/lib/scudo/CMakeLists.txt
@@ -14,6 +14,7 @@ set(SCUDO_SOURCES
   scudo_interceptors.cpp
   scudo_new_delete.cpp
   scudo_termination.cpp
+  scudo_tls_linux.cpp
   scudo_utils.cpp)
 
 # Enable the SSE 4.2 instruction set for scudo_crc32.cpp, if available.
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index e89e09223ff8..2ccdcd903dad 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "scudo_allocator.h"
+#include "scudo_tls.h"
 #include "scudo_utils.h"
 
 #include "sanitizer_common/sanitizer_allocator_interface.h"
@@ -26,44 +27,6 @@
 
 namespace __scudo {
 
-#if SANITIZER_CAN_USE_ALLOCATOR64
-const uptr AllocatorSpace = ~0ULL;
-const uptr AllocatorSize = 0x40000000000ULL;
-typedef DefaultSizeClassMap SizeClassMap;
-struct AP {
-  static const uptr kSpaceBeg = AllocatorSpace;
-  static const uptr kSpaceSize = AllocatorSize;
-  static const uptr kMetadataSize = 0;
-  typedef __scudo::SizeClassMap SizeClassMap;
-  typedef NoOpMapUnmapCallback MapUnmapCallback;
-  static const uptr kFlags =
-      SizeClassAllocator64FlagMasks::kRandomShuffleChunks;
-};
-typedef SizeClassAllocator64<AP> PrimaryAllocator;
-#else
-// Currently, the 32-bit Sanitizer allocator has not yet benefited from all the
-// security improvements brought to the 64-bit one. This makes the 32-bit
-// version of Scudo slightly less toughened.
-static const uptr RegionSizeLog = 20;
-static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog;
-# if SANITIZER_WORDSIZE == 32
-typedef FlatByteMap<NumRegions> ByteMap;
-# elif SANITIZER_WORDSIZE == 64
-typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap;
-# endif  // SANITIZER_WORDSIZE
-typedef DefaultSizeClassMap SizeClassMap;
-typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, 0, SizeClassMap,
-    RegionSizeLog, ByteMap> PrimaryAllocator;
-#endif  // SANITIZER_CAN_USE_ALLOCATOR64
-
-typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
-typedef ScudoLargeMmapAllocator SecondaryAllocator;
-typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, SecondaryAllocator>
-  ScudoBackendAllocator;
-
-static ScudoBackendAllocator &getBackendAllocator();
-
-static thread_local Xorshift128Plus Prng;
 // Global static cookie, initialized at start-up.
 static uptr Cookie;
 
@@ -88,6 +51,8 @@ INLINE u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) {
 #endif  // defined(__SSE4_2__)
 }
 
+static ScudoBackendAllocator &getBackendAllocator();
+
 struct ScudoChunk : UnpackedHeader {
   // We can't use the offset member of the chunk itself, as we would double
   // fetch it without any warranty that it wouldn't have been tampered. To
@@ -188,32 +153,44 @@ ScudoChunk *getScudoChunk(uptr UserBeg) {
   return reinterpret_cast<ScudoChunk *>(UserBeg - AlignedChunkHeaderSize);
 }
 
-static bool ScudoInitIsRunning = false;
+struct AllocatorOptions {
+  u32 QuarantineSizeMb;
+  u32 ThreadLocalQuarantineSizeKb;
+  bool MayReturnNull;
+  s32 ReleaseToOSIntervalMs;
+  bool DeallocationTypeMismatch;
+  bool DeleteSizeMismatch;
+  bool ZeroContents;
 
-static pthread_once_t GlobalInited = PTHREAD_ONCE_INIT;
-static pthread_key_t PThreadKey;
-
-static thread_local bool ThreadInited = false;
-static thread_local bool ThreadTornDown = false;
-static thread_local AllocatorCache Cache;
-
-static void teardownThread(void *p) {
-  uptr v = reinterpret_cast<uptr>(p);
-  // The glibc POSIX thread-local-storage deallocation routine calls user
-  // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS.
-  // We want to be called last since other destructors might call free and the
-  // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
-  // quarantine and swallowing the cache.
-  if (v < PTHREAD_DESTRUCTOR_ITERATIONS) {
-    pthread_setspecific(PThreadKey, reinterpret_cast<void *>(v + 1));
-    return;
-  }
-  drainQuarantine();
-  getBackendAllocator().DestroyCache(&Cache);
-  ThreadTornDown = true;
+  void setFrom(const Flags *f, const CommonFlags *cf);
+  void copyTo(Flags *f, CommonFlags *cf) const;
+};
+
+void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) {
+  MayReturnNull = cf->allocator_may_return_null;
+  ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms;
+  QuarantineSizeMb = f->QuarantineSizeMb;
+  ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb;
+  DeallocationTypeMismatch = f->DeallocationTypeMismatch;
+  DeleteSizeMismatch = f->DeleteSizeMismatch;
+  ZeroContents = f->ZeroContents;
+}
+
+void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const {
+  cf->allocator_may_return_null = MayReturnNull;
+  cf->allocator_release_to_os_interval_ms = ReleaseToOSIntervalMs;
+  f->QuarantineSizeMb = QuarantineSizeMb;
+  f->ThreadLocalQuarantineSizeKb = ThreadLocalQuarantineSizeKb;
+  f->DeallocationTypeMismatch = DeallocationTypeMismatch;
+  f->DeleteSizeMismatch = DeleteSizeMismatch;
+  f->ZeroContents = ZeroContents;
 }
 
-static void initInternal() {
+static void initScudoInternal(const AllocatorOptions &Options);
+
+static bool ScudoInitIsRunning = false;
+
+void initScudo() {
   SanitizerToolName = "Scudo";
   CHECK(!ScudoInitIsRunning && "Scudo init calls itself!");
   ScudoInitIsRunning = true;
@@ -227,25 +204,13 @@ static void initInternal() {
 
   AllocatorOptions Options;
   Options.setFrom(getFlags(), common_flags());
-  initAllocator(Options);
+  initScudoInternal(Options);
 
-  MaybeStartBackgroudThread();
+  // TODO(kostyak): determine if MaybeStartBackgroudThread could be of some use.
 
   ScudoInitIsRunning = false;
 }
 
-static void initGlobal() {
-  pthread_key_create(&PThreadKey, teardownThread);
-  initInternal();
-}
-
-static void NOINLINE initThread() {
-  pthread_once(&GlobalInited, initGlobal);
-  pthread_setspecific(PThreadKey, reinterpret_cast<void *>(1));
-  getBackendAllocator().InitCache(&Cache);
-  ThreadInited = true;
-}
-
 struct QuarantineCallback {
   explicit QuarantineCallback(AllocatorCache *Cache)
     : Cache_(Cache) {}
@@ -278,26 +243,20 @@ struct QuarantineCallback {
 
 typedef Quarantine<QuarantineCallback, ScudoChunk> ScudoQuarantine;
 typedef ScudoQuarantine::Cache ScudoQuarantineCache;
-static thread_local ScudoQuarantineCache ThreadQuarantineCache;
+COMPILER_CHECK(sizeof(ScudoQuarantineCache) <=
+               sizeof(ScudoThreadContext::QuarantineCachePlaceHolder));
 
-void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) {
-  MayReturnNull = cf->allocator_may_return_null;
-  ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms;
-  QuarantineSizeMb = f->QuarantineSizeMb;
-  ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb;
-  DeallocationTypeMismatch = f->DeallocationTypeMismatch;
-  DeleteSizeMismatch = f->DeleteSizeMismatch;
-  ZeroContents = f->ZeroContents;
+AllocatorCache *getAllocatorCache(ScudoThreadContext *ThreadContext) {
+  return &ThreadContext->Cache;
 }
 
-void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const {
-  cf->allocator_may_return_null = MayReturnNull;
-  cf->allocator_release_to_os_interval_ms = ReleaseToOSIntervalMs;
-  f->QuarantineSizeMb = QuarantineSizeMb;
-  f->ThreadLocalQuarantineSizeKb = ThreadLocalQuarantineSizeKb;
-  f->DeallocationTypeMismatch = DeallocationTypeMismatch;
-  f->DeleteSizeMismatch = DeleteSizeMismatch;
-  f->ZeroContents = ZeroContents;
+ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) {
+  return reinterpret_cast<
+      ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder);
+}
+
+Xorshift128Plus *getPrng(ScudoThreadContext *ThreadContext) {
+  return &ThreadContext->Prng;
 }
 
 struct ScudoAllocator {
@@ -313,6 +272,7 @@ struct ScudoAllocator {
   StaticSpinMutex FallbackMutex;
   AllocatorCache FallbackAllocatorCache;
   ScudoQuarantineCache FallbackQuarantineCache;
+  Xorshift128Plus FallbackPrng;
 
   bool DeallocationTypeMismatch;
   bool ZeroContents;
@@ -361,13 +321,13 @@ struct ScudoAllocator {
         static_cast<uptr>(Options.QuarantineSizeMb) << 20,
         static_cast<uptr>(Options.ThreadLocalQuarantineSizeKb) << 10);
     BackendAllocator.InitCache(&FallbackAllocatorCache);
-    Cookie = Prng.getNext();
+    FallbackPrng.initFromURandom();
+    Cookie = FallbackPrng.getNext();
   }
 
   // Helper function that checks for a valid Scudo chunk. nullptr isn't.
   bool isValidPointer(const void *UserPtr) {
-    if (UNLIKELY(!ThreadInited))
-      initThread();
+    initThreadMaybe();
     if (!UserPtr)
       return false;
     uptr UserBeg = reinterpret_cast<uptr>(UserPtr);
@@ -379,8 +339,7 @@ struct ScudoAllocator {
   // Allocates a chunk.
   void *allocate(uptr Size, uptr Alignment, AllocType Type,
                  bool ForceZeroContents = false) {
-    if (UNLIKELY(!ThreadInited))
-      initThread();
+    initThreadMaybe();
     if (UNLIKELY(!IsPowerOfTwo(Alignment))) {
       dieWithMessage("ERROR: alignment is not a power of 2\n");
     }
@@ -407,11 +366,16 @@ struct ScudoAllocator {
     bool FromPrimary = PrimaryAllocator::CanAllocate(NeededSize, MinAlignment);
 
     void *Ptr;
+    uptr Salt;
     uptr AllocationAlignment = FromPrimary ? MinAlignment : Alignment;
-    if (LIKELY(!ThreadTornDown)) {
-      Ptr = BackendAllocator.Allocate(&Cache, NeededSize, AllocationAlignment);
+    ScudoThreadContext *ThreadContext = getThreadContext();
+    if (LIKELY(ThreadContext)) {
+      Salt = getPrng(ThreadContext)->getNext();
+      Ptr = BackendAllocator.Allocate(getAllocatorCache(ThreadContext),
+                                      NeededSize, AllocationAlignment);
     } else {
       SpinMutexLock l(&FallbackMutex);
+      Salt = FallbackPrng.getNext();
       Ptr = BackendAllocator.Allocate(&FallbackAllocatorCache, NeededSize,
                                       AllocationAlignment);
     }
@@ -453,7 +417,7 @@ struct ScudoAllocator {
       if (TrailingBytes)
         Header.SizeOrUnusedBytes = PageSize - TrailingBytes;
     }
-    Header.Salt = static_cast<u8>(Prng.getNext());
+    Header.Salt = static_cast<u8>(Salt);
     getScudoChunk(UserBeg)->storeHeader(&Header);
     void *UserPtr = reinterpret_cast<void *>(UserBeg);
     // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(UserPtr, Size);
@@ -462,16 +426,17 @@ struct ScudoAllocator {
 
   // Place a chunk in the quarantine. In the event of a zero-sized quarantine,
   // we directly deallocate the chunk, otherwise the flow would lead to the
-  // chunk being checksummed twice, once before Put and once in Recycle, with
-  // no additional security value.
+  // chunk being loaded (and checked) twice, and stored (and checksummed) once,
+  // with no additional security value.
   void quarantineOrDeallocateChunk(ScudoChunk *Chunk, UnpackedHeader *Header,
                                    uptr Size) {
     bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0);
     if (BypassQuarantine) {
       Chunk->eraseHeader();
       void *Ptr = Chunk->getAllocBeg(Header);
-      if (LIKELY(!ThreadTornDown)) {
-        getBackendAllocator().Deallocate(&Cache, Ptr);
+      ScudoThreadContext *ThreadContext = getThreadContext();
+      if (LIKELY(ThreadContext)) {
+        getBackendAllocator().Deallocate(getAllocatorCache(ThreadContext), Ptr);
       } else {
         SpinMutexLock Lock(&FallbackMutex);
         getBackendAllocator().Deallocate(&FallbackAllocatorCache, Ptr);
@@ -480,9 +445,12 @@ struct ScudoAllocator {
       UnpackedHeader NewHeader = *Header;
       NewHeader.State = ChunkQuarantine;
       Chunk->compareExchangeHeader(&NewHeader, Header);
-      if (LIKELY(!ThreadTornDown)) {
-        AllocatorQuarantine.Put(&ThreadQuarantineCache,
-                                QuarantineCallback(&Cache), Chunk, Size);
+      ScudoThreadContext *ThreadContext = getThreadContext();
+      if (LIKELY(ThreadContext)) {
+        AllocatorQuarantine.Put(getQuarantineCache(ThreadContext),
+                                QuarantineCallback(
+                                    getAllocatorCache(ThreadContext)),
+                                Chunk, Size);
       } else {
         SpinMutexLock l(&FallbackMutex);
         AllocatorQuarantine.Put(&FallbackQuarantineCache,
@@ -495,8 +463,7 @@ struct ScudoAllocator {
   // Deallocates a Chunk, which means adding it to the delayed free list (or
   // Quarantine).
   void deallocate(void *UserPtr, uptr DeleteSize, AllocType Type) {
-    if (UNLIKELY(!ThreadInited))
-      initThread();
+    initThreadMaybe();
     // if (&__sanitizer_free_hook) __sanitizer_free_hook(UserPtr);
     if (!UserPtr)
       return;
@@ -542,8 +509,7 @@ struct ScudoAllocator {
   // Reallocates a chunk. We can save on a new allocation if the new requested
   // size still fits in the chunk.
   void *reallocate(void *OldPtr, uptr NewSize) {
-    if (UNLIKELY(!ThreadInited))
-      initThread();
+    initThreadMaybe();
     uptr UserBeg = reinterpret_cast<uptr>(OldPtr);
     if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) {
       dieWithMessage("ERROR: attempted to reallocate a chunk not properly "
@@ -585,8 +551,7 @@ struct ScudoAllocator {
 
   // Helper function that returns the actual usable size of a chunk.
   uptr getUsableSize(const void *Ptr) {
-    if (UNLIKELY(!ThreadInited))
-      initThread();
+    initThreadMaybe();
     if (!Ptr)
       return 0;
     uptr UserBeg = reinterpret_cast<uptr>(Ptr);
@@ -602,22 +567,22 @@ struct ScudoAllocator {
   }
 
   void *calloc(uptr NMemB, uptr Size) {
-    if (UNLIKELY(!ThreadInited))
-      initThread();
+    initThreadMaybe();
     uptr Total = NMemB * Size;
     if (Size != 0 && Total / Size != NMemB)  // Overflow check
       return BackendAllocator.ReturnNullOrDieOnBadRequest();
     return allocate(Total, MinAlignment, FromMalloc, true);
   }
 
-  void drainQuarantine() {
-    AllocatorQuarantine.Drain(&ThreadQuarantineCache,
-                              QuarantineCallback(&Cache));
+  void commitBack(ScudoThreadContext *ThreadContext) {
+    AllocatorCache *Cache = getAllocatorCache(ThreadContext);
+    AllocatorQuarantine.Drain(getQuarantineCache(ThreadContext),
+                              QuarantineCallback(Cache));
+    BackendAllocator.DestroyCache(Cache);
   }
 
   uptr getStats(AllocatorStat StatType) {
-    if (UNLIKELY(!ThreadInited))
-      initThread();
+    initThreadMaybe();
     uptr stats[AllocatorStatCount];
     BackendAllocator.GetStats(stats);
     return stats[StatType];
@@ -630,12 +595,18 @@ static ScudoBackendAllocator &getBackendAllocator() {
   return Instance.BackendAllocator;
 }
 
-void initAllocator(const AllocatorOptions &Options) {
+static void initScudoInternal(const AllocatorOptions &Options) {
   Instance.init(Options);
 }
 
-void drainQuarantine() {
-  Instance.drainQuarantine();
+void ScudoThreadContext::init() {
+  getBackendAllocator().InitCache(&Cache);
+  Prng.initFromURandom();
+  memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
+}
+
+void ScudoThreadContext::commitBack() {
+  Instance.commitBack(this);
 }
 
 void *scudoMalloc(uptr Size, AllocType Type) {
diff --git a/lib/scudo/scudo_allocator.h b/lib/scudo/scudo_allocator.h
index e7428f170271..2cac2de71cb0 100644
--- a/lib/scudo/scudo_allocator.h
+++ b/lib/scudo/scudo_allocator.h
@@ -53,7 +53,7 @@ struct UnpackedHeader {
   u64 Offset            : 16; // Offset from the beginning of the backend
                               // allocation to the beginning of the chunk
                               // itself, in multiples of MinAlignment. See
-                              /// comment about its maximum value and in init().
+                              // comment about its maximum value and in init().
   u64 Salt              : 8;
 };
 
@@ -62,7 +62,7 @@ COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader));
 
 // Minimum alignment of 8 bytes for 32-bit, 16 for 64-bit
 const uptr MinAlignmentLog = FIRST_32_SECOND_64(3, 4);
-const uptr MaxAlignmentLog = 24; // 16 MB
+const uptr MaxAlignmentLog = 24;  // 16 MB
 const uptr MinAlignment = 1 << MinAlignmentLog;
 const uptr MaxAlignment = 1 << MaxAlignmentLog;
 
@@ -70,21 +70,44 @@ const uptr ChunkHeaderSize = sizeof(PackedHeader);
 const uptr AlignedChunkHeaderSize =
     (ChunkHeaderSize + MinAlignment - 1) & ~(MinAlignment - 1);
 
-struct AllocatorOptions {
-  u32 QuarantineSizeMb;
-  u32 ThreadLocalQuarantineSizeKb;
-  bool MayReturnNull;
-  s32 ReleaseToOSIntervalMs;
-  bool DeallocationTypeMismatch;
-  bool DeleteSizeMismatch;
-  bool ZeroContents;
-
-  void setFrom(const Flags *f, const CommonFlags *cf);
-  void copyTo(Flags *f, CommonFlags *cf) const;
+#if SANITIZER_CAN_USE_ALLOCATOR64
+const uptr AllocatorSpace = ~0ULL;
+const uptr AllocatorSize = 0x40000000000ULL;  // 4TB.
+typedef DefaultSizeClassMap SizeClassMap;
+struct AP {
+  static const uptr kSpaceBeg = AllocatorSpace;
+  static const uptr kSpaceSize = AllocatorSize;
+  static const uptr kMetadataSize = 0;
+  typedef __scudo::SizeClassMap SizeClassMap;
+  typedef NoOpMapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags =
+      SizeClassAllocator64FlagMasks::kRandomShuffleChunks;
 };
+typedef SizeClassAllocator64<AP> PrimaryAllocator;
+#else
+// Currently, the 32-bit Sanitizer allocator has not yet benefited from all the
+// security improvements brought to the 64-bit one. This makes the 32-bit
+// version of Scudo slightly less toughened.
+static const uptr RegionSizeLog = 20;
+static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog;
+# if SANITIZER_WORDSIZE == 32
+typedef FlatByteMap<NumRegions> ByteMap;
+# elif SANITIZER_WORDSIZE == 64
+typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap;
+# endif  // SANITIZER_WORDSIZE
+typedef DefaultSizeClassMap SizeClassMap;
+typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, 0, SizeClassMap,
+    RegionSizeLog, ByteMap> PrimaryAllocator;
+#endif  // SANITIZER_CAN_USE_ALLOCATOR64
 
-void initAllocator(const AllocatorOptions &options);
-void drainQuarantine();
+#include "scudo_allocator_secondary.h"
+
+typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
+typedef ScudoLargeMmapAllocator SecondaryAllocator;
+typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, SecondaryAllocator>
+    ScudoBackendAllocator;
+
+void initScudo();
 
 void *scudoMalloc(uptr Size, AllocType Type);
 void scudoFree(void *Ptr, AllocType Type);
@@ -98,8 +121,6 @@ int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size);
 void *scudoAlignedAlloc(uptr Alignment, uptr Size);
 uptr scudoMallocUsableSize(void *Ptr);
 
-#include "scudo_allocator_secondary.h"
-
 }  // namespace __scudo
 
 #endif  // SCUDO_ALLOCATOR_H_
diff --git a/lib/scudo/scudo_tls.h b/lib/scudo/scudo_tls.h
new file mode 100644
index 000000000000..0d7d1bffd0b6
--- /dev/null
+++ b/lib/scudo/scudo_tls.h
@@ -0,0 +1,40 @@
+//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo thread local structure definition.
+/// Implementation will differ based on the thread local storage primitives
+/// offered by the underlying platform.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TLS_H_
+#define SCUDO_TLS_H_
+
+#include "scudo_allocator.h"
+#include "scudo_utils.h"
+
+namespace __scudo {
+
+struct ALIGNED(64) ScudoThreadContext {
+ public:
+  AllocatorCache Cache;
+  Xorshift128Plus Prng;
+  uptr QuarantineCachePlaceHolder[4];
+  void init();
+  void commitBack();
+};
+
+void initThread();
+
+// Fastpath functions are defined in the following platform specific headers.
+#include "scudo_tls_linux.h"
+
+}  // namespace __scudo
+
+#endif  // SCUDO_TLS_H_
diff --git a/lib/scudo/scudo_tls_linux.cpp b/lib/scudo/scudo_tls_linux.cpp
new file mode 100644
index 000000000000..3453367f8a53
--- /dev/null
+++ b/lib/scudo/scudo_tls_linux.cpp
@@ -0,0 +1,62 @@
+//===-- scudo_tls_linux.cpp -------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo thread local structure implementation for platforms supporting
+/// thread_local.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if SANITIZER_LINUX
+
+#include "scudo_tls.h"
+
+#include <limits.h>
+#include <pthread.h>
+
+namespace __scudo {
+
+static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
+static pthread_key_t PThreadKey;
+
+thread_local ThreadState ScudoThreadState = ThreadNotInitialized;
+thread_local ScudoThreadContext ThreadLocalContext;
+
+static void teardownThread(void *Ptr) {
+  uptr Iteration = reinterpret_cast<uptr>(Ptr);
+  // The glibc POSIX thread-local-storage deallocation routine calls user
+  // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS.
+  // We want to be called last since other destructors might call free and the
+  // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
+  // quarantine and swallowing the cache.
+  if (Iteration < PTHREAD_DESTRUCTOR_ITERATIONS) {
+    pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Iteration + 1));
+    return;
+  }
+  ThreadLocalContext.commitBack();
+  ScudoThreadState = ThreadTornDown;
+}
+
+
+static void initOnce() {
+  CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0);
+  initScudo();
+}
+
+void initThread() {
+  pthread_once(&GlobalInitialized, initOnce);
+  pthread_setspecific(PThreadKey, reinterpret_cast<void *>(1));
+  ThreadLocalContext.init();
+  ScudoThreadState = ThreadInitialized;
+}
+
+}  // namespace __scudo
+
+#endif  // SANITIZER_LINUX
diff --git a/lib/scudo/scudo_tls_linux.h b/lib/scudo/scudo_tls_linux.h
new file mode 100644
index 000000000000..0994f2d7b24d
--- /dev/null
+++ b/lib/scudo/scudo_tls_linux.h
@@ -0,0 +1,48 @@
+//===-- scudo_tls_linux.h ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo thread local structure fastpath functions implementation for platforms
+/// supporting thread_local.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TLS_LINUX_H_
+#define SCUDO_TLS_LINUX_H_
+
+#ifndef SCUDO_TLS_H_
+# error "This file must be included inside scudo_tls.h."
+#endif  // SCUDO_TLS_H_
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if SANITIZER_LINUX
+
+enum ThreadState : u8 {
+  ThreadNotInitialized = 0,
+  ThreadInitialized,
+  ThreadTornDown,
+};
+extern thread_local ThreadState ScudoThreadState;
+extern thread_local ScudoThreadContext ThreadLocalContext;
+
+ALWAYS_INLINE void initThreadMaybe() {
+  if (LIKELY(ScudoThreadState != ThreadNotInitialized))
+    return;
+  initThread();
+}
+
+ALWAYS_INLINE ScudoThreadContext *getThreadContext() {
+  if (UNLIKELY(ScudoThreadState == ThreadTornDown))
+    return nullptr;
+  return &ThreadLocalContext;
+}
+
+#endif  // SANITIZER_LINUX
+
+#endif  // SCUDO_TLS_LINUX_H_
diff --git a/lib/scudo/scudo_utils.cpp b/lib/scudo/scudo_utils.cpp
index 98bd591aa868..31c391946c15 100644
--- a/lib/scudo/scudo_utils.cpp
+++ b/lib/scudo/scudo_utils.cpp
@@ -153,9 +153,9 @@ static void fillRandom(u8 *Data, ssize_t Size) {
   }
 }
 
-// Default constructor for Xorshift128Plus seeds the state with /dev/urandom.
+// Seeds the xorshift state with /dev/urandom.
 // TODO(kostyak): investigate using getrandom() if available.
-Xorshift128Plus::Xorshift128Plus() {
+void Xorshift128Plus::initFromURandom() {
   fillRandom(reinterpret_cast<u8 *>(State), sizeof(State));
 }
 
diff --git a/lib/scudo/scudo_utils.h b/lib/scudo/scudo_utils.h
index f30c86125799..484b0c859e3d 100644
--- a/lib/scudo/scudo_utils.h
+++ b/lib/scudo/scudo_utils.h
@@ -40,7 +40,7 @@ bool testCPUFeature(CPUFeature feature);
 // The state (128 bits) will be stored in thread local storage.
 struct Xorshift128Plus {
  public:
-  Xorshift128Plus();
+  void initFromURandom();
   u64 getNext() {
     u64 x = State[0];
     const u64 y = State[1];
diff --git a/lib/tsan/go/buildgo.sh b/lib/tsan/go/buildgo.sh
index 42d479064c49..59176809eae0 100755
--- a/lib/tsan/go/buildgo.sh
+++ b/lib/tsan/go/buildgo.sh
@@ -5,6 +5,7 @@ set -e
 SRCS="
 	tsan_go.cc
 	../rtl/tsan_clock.cc
+	../rtl/tsan_external.cc
 	../rtl/tsan_flags.cc
 	../rtl/tsan_interface_atomic.cc
 	../rtl/tsan_md5.cc
diff --git a/lib/tsan/rtl/tsan_external.cc b/lib/tsan/rtl/tsan_external.cc
index 88468e406651..2d32b6dac75a 100644
--- a/lib/tsan/rtl/tsan_external.cc
+++ b/lib/tsan/rtl/tsan_external.cc
@@ -17,11 +17,8 @@ namespace __tsan {
 
 #define CALLERPC ((uptr)__builtin_return_address(0))
 
-const uptr kMaxTag = 128;  // Limited to 65,536, since MBlock only stores tags
-                           // as 16-bit values, see tsan_defs.h.
-
-const char *registered_tags[kMaxTag];
-static atomic_uint32_t used_tags{1};  // Tag 0 means "no tag". NOLINT
+const char *registered_tags[kExternalTagMax];
+static atomic_uint32_t used_tags{kExternalTagFirstUserAvailable};  // NOLINT.
 
 const char *GetObjectTypeFromTag(uptr tag) {
   if (tag == 0) return nullptr;
@@ -30,25 +27,39 @@ const char *GetObjectTypeFromTag(uptr tag) {
   return registered_tags[tag];
 }
 
+void InsertShadowStackFrameForTag(ThreadState *thr, uptr tag) {
+  FuncEntry(thr, (uptr)&registered_tags[tag]);
+}
+
+uptr TagFromShadowStackFrame(uptr pc) {
+  uptr tag_count = atomic_load(&used_tags, memory_order_relaxed);
+  void *pc_ptr = (void *)pc;
+  if (pc_ptr < &registered_tags[0] || pc_ptr >= &registered_tags[tag_count])
+    return 0;
+  return (const char **)pc_ptr - &registered_tags[0];
+}
+
+#if !SANITIZER_GO
+
 typedef void(*AccessFunc)(ThreadState *, uptr, uptr, int);
 void ExternalAccess(void *addr, void *caller_pc, void *tag, AccessFunc access) {
   CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
   ThreadState *thr = cur_thread();
-  thr->external_tag = (uptr)tag;
   if (caller_pc) FuncEntry(thr, (uptr)caller_pc);
+  InsertShadowStackFrameForTag(thr, (uptr)tag);
   bool in_ignored_lib;
   if (!caller_pc || !libignore()->IsIgnored((uptr)caller_pc, &in_ignored_lib)) {
     access(thr, CALLERPC, (uptr)addr, kSizeLog1);
   }
+  FuncExit(thr);
   if (caller_pc) FuncExit(thr);
-  thr->external_tag = 0;
 }
 
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__tsan_external_register_tag(const char *object_type) {
   uptr new_tag = atomic_fetch_add(&used_tags, 1, memory_order_relaxed);
-  CHECK_LT(new_tag, kMaxTag);
+  CHECK_LT(new_tag, kExternalTagMax);
   registered_tags[new_tag] = internal_strdup(object_type);
   return (void *)new_tag;
 }
@@ -78,4 +89,6 @@ void __tsan_external_write(void *addr, void *caller_pc, void *tag) {
 }
 }  // extern "C"
 
+#endif  // !SANITIZER_GO
+
 }  // namespace __tsan
diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc
index 810c84025f23..45ec45bbdbbe 100644
--- a/lib/tsan/rtl/tsan_interface_ann.cc
+++ b/lib/tsan/rtl/tsan_interface_ann.cc
@@ -471,7 +471,7 @@ void __tsan_mutex_create(void *m, unsigned flagz) {
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_destroy(void *m, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_destroy);
-  MutexDestroy(thr, pc, (uptr)m);
+  MutexDestroy(thr, pc, (uptr)m, flagz);
 }
 
 INTERFACE_ATTRIBUTE
diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc
index 3313288a728a..2d488cadd4fe 100644
--- a/lib/tsan/rtl/tsan_platform_linux.cc
+++ b/lib/tsan/rtl/tsan_platform_linux.cc
@@ -341,36 +341,22 @@ void ReplaceSystemMalloc() { }
 
 #if !SANITIZER_GO
 #if SANITIZER_ANDROID
-
-#if defined(__aarch64__)
-# define __get_tls() \
-    ({ void** __val; __asm__("mrs %0, tpidr_el0" : "=r"(__val)); __val; })
-#elif defined(__x86_64__)
-# define __get_tls() \
-    ({ void** __val; __asm__("mov %%fs:0, %0" : "=r"(__val)); __val; })
-#else
-#error unsupported architecture
-#endif
-
-// On Android, __thread is not supported. So we store the pointer to ThreadState
-// in TLS_SLOT_TSAN, which is the tls slot allocated by Android bionic for tsan.
-static const int TLS_SLOT_TSAN = 8;
 // On Android, one thread can call intercepted functions after
 // DestroyThreadState(), so add a fake thread state for "dead" threads.
 static ThreadState *dead_thread_state = nullptr;
 
 ThreadState *cur_thread() {
-  ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN];
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
   if (thr == nullptr) {
     __sanitizer_sigset_t emptyset;
     internal_sigfillset(&emptyset);
     __sanitizer_sigset_t oldset;
     CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
-    thr = reinterpret_cast<ThreadState*>(__get_tls()[TLS_SLOT_TSAN]);
+    thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
     if (thr == nullptr) {
       thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState),
                                                      "ThreadState"));
-      __get_tls()[TLS_SLOT_TSAN] = thr;
+      *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
       if (dead_thread_state == nullptr) {
         dead_thread_state = reinterpret_cast<ThreadState*>(
             MmapOrDie(sizeof(ThreadState), "ThreadState"));
@@ -392,9 +378,9 @@ void cur_thread_finalize() {
   internal_sigfillset(&emptyset);
   __sanitizer_sigset_t oldset;
   CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
-  ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN];
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
   if (thr != dead_thread_state) {
-    __get_tls()[TLS_SLOT_TSAN] = dead_thread_state;
+    *get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state);
     UnmapOrDie(thr, sizeof(ThreadState));
   }
   CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
diff --git a/lib/tsan/rtl/tsan_report.cc b/lib/tsan/rtl/tsan_report.cc
index af5fe61761d7..b188af2a0d48 100644
--- a/lib/tsan/rtl/tsan_report.cc
+++ b/lib/tsan/rtl/tsan_report.cc
@@ -164,7 +164,7 @@ static void PrintMop(const ReportMop *mop, bool first) {
   char thrbuf[kThreadBufSize];
   Printf("%s", d.Access());
   const char *object_type = GetObjectTypeFromTag(mop->external_tag);
-  if (!object_type) {
+  if (mop->external_tag == kExternalTagNone || !object_type) {
     Printf("  %s of size %d at %p by %s",
            MopDesc(first, mop->write, mop->atomic), mop->size,
            (void *)mop->addr, thread_name(thrbuf, mop->tid));
diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index 70393037e786..fa60f3247c38 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -866,9 +866,8 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
   // Don't want to touch lots of shadow memory.
   // If a program maps 10MB stack, there is no need reset the whole range.
   size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
-  // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
-  // so we do it only for C/C++.
-  if (SANITIZER_GO || size < common_flags()->clear_shadow_mmap_threshold) {
+  // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
+  if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
     u64 *p = (u64*)MemToShadow(addr);
     CHECK(IsShadowMem((uptr)p));
     CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h
index 09c97a3a4f3d..8bf1c191ad76 100644
--- a/lib/tsan/rtl/tsan_rtl.h
+++ b/lib/tsan/rtl/tsan_rtl.h
@@ -411,7 +411,6 @@ struct ThreadState {
   bool is_dead;
   bool is_freeing;
   bool is_vptr_access;
-  uptr external_tag;
   const uptr stk_addr;
   const uptr stk_size;
   const uptr tls_addr;
@@ -565,6 +564,16 @@ struct ScopedIgnoreInterceptors {
   }
 };
 
+enum ExternalTag : uptr {
+  kExternalTagNone = 0,
+  kExternalTagFirstUserAvailable = 1,
+  kExternalTagMax = 1024,
+  // Don't set kExternalTagMax over 65,536, since MBlock only stores tags
+  // as 16-bit values, see tsan_defs.h.
+};
+const char *GetObjectTypeFromTag(uptr tag);
+uptr TagFromShadowStackFrame(uptr pc);
+
 class ScopedReport {
  public:
   explicit ScopedReport(ReportType typ);
@@ -598,10 +607,26 @@ class ScopedReport {
 
 ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
 void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
-                  MutexSet *mset);
+                  MutexSet *mset, uptr *tag = nullptr);
+
+// The stack could look like:
+//   <start> | <main> | <foo> | tag | <bar>
+// This will extract the tag and keep:
+//   <start> | <main> | <foo> | <bar>
+template<typename StackTraceTy>
+void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
+  if (stack->size < 2) return;
+  uptr possible_tag_pc = stack->trace[stack->size - 2];
+  uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
+  if (possible_tag == kExternalTagNone) return;
+  stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
+  stack->size -= 1;
+  if (tag) *tag = possible_tag;
+}
 
 template<typename StackTraceTy>
-void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) {
+void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
+                        uptr *tag = nullptr) {
   uptr size = thr->shadow_stack_pos - thr->shadow_stack;
   uptr start = 0;
   if (size + !!toppc > kStackTraceMax) {
@@ -609,6 +634,7 @@ void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) {
     size = kStackTraceMax - !!toppc;
   }
   stack->Init(&thr->shadow_stack[start], size, toppc);
+  ExtractTagFromStack(stack, tag);
 }
 
 
@@ -646,8 +672,6 @@ bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
 bool IsExpectedReport(uptr addr, uptr size);
 void PrintMatchedBenignRaces();
 
-const char *GetObjectTypeFromTag(uptr tag);
-
 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
 # define DPrintf Printf
 #else
@@ -739,7 +763,7 @@ void ProcUnwire(Processor *proc, ThreadState *thr);
 // Note: the parameter is called flagz, because flags is already taken
 // by the global function that returns flags.
 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
-void MutexDestroy(ThreadState *thr, uptr pc, uptr addr);
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
     int rec = 1);
diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc
index 086b28927919..54938f37e243 100644
--- a/lib/tsan/rtl/tsan_rtl_mutex.cc
+++ b/lib/tsan/rtl/tsan_rtl_mutex.cc
@@ -78,13 +78,13 @@ void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   s->mtx.Unlock();
 }
 
-void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) {
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
   StatInc(thr, StatMutexDestroy);
   SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
   if (s == 0)
     return;
-  if (s->IsFlagSet(MutexFlagLinkerInit)) {
+  if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit)) {
     // Destroy is no-op for linker-initialized mutexes.
     s->mtx.Unlock();
     return;
diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc
index 5cd93a184ce7..055029b91f5b 100644
--- a/lib/tsan/rtl/tsan_rtl_report.cc
+++ b/lib/tsan/rtl/tsan_rtl_report.cc
@@ -377,7 +377,7 @@ const ReportDesc *ScopedReport::GetReport() const {
 }
 
 void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
-                  MutexSet *mset) {
+                  MutexSet *mset, uptr *tag) {
   // This function restores stack trace and mutex set for the thread/epoch.
   // It does so by getting stack trace and mutex set at the beginning of
   // trace part, and then replaying the trace till the given epoch.
@@ -436,6 +436,7 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
     return;
   pos++;
   stk->Init(&stack[0], pos);
+  ExtractTagFromStack(stk, tag);
 }
 
 static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2],
@@ -625,16 +626,15 @@ void ReportRace(ThreadState *thr) {
     typ = ReportTypeVptrRace;
   else if (freed)
     typ = ReportTypeUseAfterFree;
-  else if (thr->external_tag > 0)
-    typ = ReportTypeExternalRace;
 
   if (IsFiredSuppression(ctx, typ, addr))
     return;
 
   const uptr kMop = 2;
   VarSizeStackTrace traces[kMop];
+  uptr tags[kMop] = {kExternalTagNone};
   const uptr toppc = TraceTopPC(thr);
-  ObtainCurrentStack(thr, toppc, &traces[0]);
+  ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
   if (IsFiredSuppression(ctx, typ, traces[0]))
     return;
 
@@ -644,18 +644,22 @@ void ReportRace(ThreadState *thr) {
   MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
 
   Shadow s2(thr->racy_state[1]);
-  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2);
+  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
   if (IsFiredSuppression(ctx, typ, traces[1]))
     return;
 
   if (HandleRacyStacks(thr, traces, addr_min, addr_max))
     return;
 
+  // If any of the two accesses has a tag, treat this as an "external" race.
+  if (tags[0] != kExternalTagNone || tags[1] != kExternalTagNone)
+    typ = ReportTypeExternalRace;
+
   ThreadRegistryLock l0(ctx->thread_registry);
   ScopedReport rep(typ);
   for (uptr i = 0; i < kMop; i++) {
     Shadow s(thr->racy_state[i]);
-    rep.AddMemoryAccess(addr, thr->external_tag, s, traces[i],
+    rep.AddMemoryAccess(addr, tags[i], s, traces[i],
                         i == 0 ? &thr->mset : mset2);
   }
 
diff --git a/lib/tsan/tests/rtl/tsan_posix.cc b/lib/tsan/tests/rtl/tsan_posix.cc
index 9c0e013e5735..e66dab609c62 100644
--- a/lib/tsan/tests/rtl/tsan_posix.cc
+++ b/lib/tsan/tests/rtl/tsan_posix.cc
@@ -94,8 +94,9 @@ TEST(Posix, ThreadLocalAccesses) {
 // The test is failing with high thread count for aarch64.
 // FIXME: track down the issue and re-enable the test.
 // On Darwin, we're running unit tests without interceptors and __thread is
-// using malloc and free, which causes false data race reports.
-#if !defined(__aarch64__) && !defined(__APPLE__)
+// using malloc and free, which causes false data race reports.  On rare
+// occasions on powerpc64le this test also fails.
+#if !defined(__aarch64__) && !defined(__APPLE__) && !defined(powerpc64le)
   local_thread((void*)2);
 #endif
 }
diff --git a/lib/ubsan/ubsan_diag.cc b/lib/ubsan/ubsan_diag.cc
index bbe1e07390ad..742802b8f341 100644
--- a/lib/ubsan/ubsan_diag.cc
+++ b/lib/ubsan/ubsan_diag.cc
@@ -31,15 +31,16 @@ static void MaybePrintStackTrace(uptr pc, uptr bp) {
   // will definitely be called when we print the first diagnostics message.
   if (!flags()->print_stacktrace)
     return;
-  // We can only use slow unwind, as we don't have any information about stack
-  // top/bottom.
-  // FIXME: It's better to respect "fast_unwind_on_fatal" runtime flag and
-  // fetch stack top/bottom information if we have it (e.g. if we're running
-  // under ASan).
-  if (StackTrace::WillUseFastUnwind(false))
-    return;
+
+  uptr top = 0;
+  uptr bottom = 0;
+  bool request_fast_unwind = common_flags()->fast_unwind_on_fatal;
+  if (request_fast_unwind)
+    __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom);
+
   BufferedStackTrace stack;
-  stack.Unwind(kStackTraceMax, pc, bp, 0, 0, 0, false);
+  stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom,
+               request_fast_unwind);
   stack.Print();
 }
 
diff --git a/lib/ubsan/ubsan_handlers.cc b/lib/ubsan/ubsan_handlers.cc
index 4e025a8ddddd..de13ab893bec 100644
--- a/lib/ubsan/ubsan_handlers.cc
+++ b/lib/ubsan/ubsan_handlers.cc
@@ -390,7 +390,7 @@ static void handleFloatCastOverflow(void *DataPtr, ValueHandle From,
   ScopedReport R(Opts, Loc, ET);
 
   Diag(Loc, DL_Error,
-       "value %0 is outside the range of representable values of type %2")
+       "%0 is outside the range of representable values of type %2")
       << Value(*FromType, From) << *FromType << *ToType;
 }
 
diff --git a/lib/xray/xray_log_interface.cc b/lib/xray/xray_log_interface.cc
index ffed601c05c6..ee14ae4b1b62 100644
--- a/lib/xray/xray_log_interface.cc
+++ b/lib/xray/xray_log_interface.cc
@@ -27,12 +27,22 @@ void __xray_set_log_impl(XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT {
       Impl.handle_arg0 == nullptr || Impl.flush_log == nullptr) {
     __sanitizer::SpinMutexLock Guard(&XRayImplMutex);
     GlobalXRayImpl.reset();
+    __xray_remove_handler();
+    __xray_remove_handler_arg1();
     return;
   }
 
   __sanitizer::SpinMutexLock Guard(&XRayImplMutex);
   GlobalXRayImpl.reset(new XRayLogImpl);
   *GlobalXRayImpl = Impl;
+  __xray_set_handler(Impl.handle_arg0);
+}
+
+void __xray_remove_log_impl() XRAY_NEVER_INSTRUMENT {
+  __sanitizer::SpinMutexLock Guard(&XRayImplMutex);
+  GlobalXRayImpl.reset();
+  __xray_remove_handler();
+  __xray_remove_handler_arg1();
 }
 
 XRayLogInitStatus __xray_log_init(size_t BufferSize, size_t MaxBuffers,