1 files changed, 59 insertions, 34 deletions

diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index 629871ef8f7a..804f3cf64ee8 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -44,7 +44,8 @@ extern "C" void __tsan_resume() {
 
 namespace __tsan {
 
-#if !defined(SANITIZER_GO) && !SANITIZER_MAC
+#if !SANITIZER_GO && !SANITIZER_MAC
+__attribute__((tls_model("initial-exec")))
 THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
 #endif
 static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
@@ -86,7 +87,7 @@ static ThreadContextBase *CreateThreadContext(u32 tid) {
   return new(mem) ThreadContext(tid);
 }
 
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
 static const u32 kThreadQuarantineSize = 16;
 #else
 static const u32 kThreadQuarantineSize = 64;
@@ -117,7 +118,7 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
   // , ignore_reads_and_writes()
   // , ignore_interceptors()
   , clock(tid, reuse_count)
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   , jmp_bufs(MBlockJmpBuf)
 #endif
   , tid(tid)
@@ -126,13 +127,13 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
   , stk_size(stk_size)
   , tls_addr(tls_addr)
   , tls_size(tls_size)
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   , last_sleep_clock(tid)
 #endif
 {
 }
 
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
 static void MemoryProfiler(Context *ctx, fd_t fd, int i) {
   uptr n_threads;
   uptr n_running_threads;
@@ -233,16 +234,17 @@ static void StopBackgroundThread() {
 #endif
 
 void DontNeedShadowFor(uptr addr, uptr size) {
-  uptr shadow_beg = MemToShadow(addr);
-  uptr shadow_end = MemToShadow(addr + size);
-  FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg);
+  ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
 }
 
 void MapShadow(uptr addr, uptr size) {
   // Global data is not 64K aligned, but there are no adjacent mappings,
   // so we can get away with unaligned mapping.
   // CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
-  MmapFixedNoReserve(MemToShadow(addr), size * kShadowMultiplier, "shadow");
+  const uptr kPageSize = GetPageSizeCached();
+  uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
+  uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
+  MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow");
 
   // Meta shadow is 2:1, so tread carefully.
   static bool data_mapped = false;
@@ -287,10 +289,15 @@ void MapThreadTrace(uptr addr, uptr size, const char *name) {
 static void CheckShadowMapping() {
   uptr beg, end;
   for (int i = 0; GetUserRegion(i, &beg, &end); i++) {
+    // Skip cases for empty regions (heap definition for architectures that
+    // do not use 64-bit allocator).
+    if (beg == end)
+      continue;
     VPrintf(3, "checking shadow region %p-%p\n", beg, end);
+    uptr prev = 0;
     for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
-      for (int x = -1; x <= 1; x++) {
-        const uptr p = p0 + x;
+      for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
+        const uptr p = RoundDown(p0 + x, kShadowCell);
         if (p < beg || p >= end)
           continue;
         const uptr s = MemToShadow(p);
@@ -298,8 +305,18 @@ static void CheckShadowMapping() {
         VPrintf(3, "  checking pointer %p: shadow=%p meta=%p\n", p, s, m);
         CHECK(IsAppMem(p));
         CHECK(IsShadowMem(s));
-        CHECK_EQ(p & ~(kShadowCell - 1), ShadowToMem(s));
+        CHECK_EQ(p, ShadowToMem(s));
         CHECK(IsMetaMem(m));
+        if (prev) {
+          // Ensure that shadow and meta mappings are linear within a single
+          // user range. Lots of code that processes memory ranges assumes it.
+          const uptr prev_s = MemToShadow(prev);
+          const uptr prev_m = (uptr)MemToMeta(prev);
+          CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
+          CHECK_EQ((m - prev_m) / kMetaShadowSize,
+                   (p - prev) / kMetaShadowCell);
+        }
+        prev = p;
       }
     }
   }
@@ -318,12 +335,12 @@ void Initialize(ThreadState *thr) {
   SetCheckFailedCallback(TsanCheckFailed);
 
   ctx = new(ctx_placeholder) Context;
-  const char *options = GetEnv(kTsanOptionsEnv);
+  const char *options = GetEnv(SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS");
   CacheBinaryName();
   InitializeFlags(&ctx->flags, options);
   AvoidCVE_2016_2143();
   InitializePlatformEarly();
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   // Re-exec ourselves if we need to set additional env or command line args.
   MaybeReexec();
 
@@ -339,14 +356,14 @@ void Initialize(ThreadState *thr) {
   InitializePlatform();
   InitializeMutex();
   InitializeDynamicAnnotations();
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   InitializeShadowMemory();
   InitializeAllocatorLate();
 #endif
   // Setup correct file descriptor for error reports.
   __sanitizer_set_report_path(common_flags()->log_path);
   InitializeSuppressions();
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   InitializeLibIgnore();
   Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
   // On MIPS, TSan initialization is run before
@@ -370,7 +387,7 @@ void Initialize(ThreadState *thr) {
 #endif
   ctx->initialized = true;
 
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   Symbolizer::LateInitialize();
 #endif
 
@@ -396,7 +413,7 @@ int Finalize(ThreadState *thr) {
   CommonSanitizerReportMutex.Unlock();
   ctx->report_mtx.Unlock();
 
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   if (Verbosity()) AllocatorPrintStats();
 #endif
 
@@ -404,7 +421,7 @@ int Finalize(ThreadState *thr) {
 
   if (ctx->nreported) {
     failed = true;
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
     Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported);
 #else
     Printf("Found %d data race(s)\n", ctx->nreported);
@@ -419,7 +436,7 @@ int Finalize(ThreadState *thr) {
 
   if (common_flags()->print_suppressions)
     PrintMatchedSuppressions();
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   if (flags()->print_benign)
     PrintMatchedBenignRaces();
 #endif
@@ -434,7 +451,7 @@ int Finalize(ThreadState *thr) {
   return failed ? common_flags()->exitcode : 0;
 }
 
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
 void ForkBefore(ThreadState *thr, uptr pc) {
   ctx->thread_registry->Lock();
   ctx->report_mtx.Lock();
@@ -467,7 +484,7 @@ void ForkChildAfter(ThreadState *thr, uptr pc) {
 }
 #endif
 
-#ifdef SANITIZER_GO
+#if SANITIZER_GO
 NOINLINE
 void GrowShadowStack(ThreadState *thr) {
   const int sz = thr->shadow_stack_end - thr->shadow_stack;
@@ -486,7 +503,7 @@ u32 CurrentStackId(ThreadState *thr, uptr pc) {
   if (!thr->is_inited)  // May happen during bootstrap.
     return 0;
   if (pc != 0) {
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
     DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 #else
     if (thr->shadow_stack_pos == thr->shadow_stack_end)
@@ -532,7 +549,7 @@ uptr TraceParts() {
   return TraceSize() / kTracePartSize;
 }
 
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
 extern "C" void __tsan_trace_switch() {
   TraceSwitch(cur_thread());
 }
@@ -565,7 +582,7 @@ void HandleRace(ThreadState *thr, u64 *shadow_mem,
   thr->racy_state[0] = cur.raw();
   thr->racy_state[1] = old.raw();
   thr->racy_shadow_addr = shadow_mem;
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   HACKY_CALL(__tsan_report_race);
 #else
   ReportRace(thr);
@@ -762,7 +779,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
   }
 #endif
 
-  if (kCppMode && *shadow_mem == kShadowRodata) {
+  if (!SANITIZER_GO && *shadow_mem == kShadowRodata) {
     // Access to .rodata section, no races here.
     // Measurements show that it can be 10-20% of all memory accesses.
     StatInc(thr, StatMop);
@@ -849,7 +866,7 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
   size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
   // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
   // so we do it only for C/C++.
-  if (kGoMode || size < common_flags()->clear_shadow_mmap_threshold) {
+  if (SANITIZER_GO || size < common_flags()->clear_shadow_mmap_threshold) {
     u64 *p = (u64*)MemToShadow(addr);
     CHECK(IsShadowMem((uptr)p));
     CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
@@ -935,7 +952,7 @@ void FuncEntry(ThreadState *thr, uptr pc) {
   // Shadow stack maintenance can be replaced with
   // stack unwinding during trace switch (which presumably must be faster).
   DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 #else
   if (thr->shadow_stack_pos == thr->shadow_stack_end)
@@ -955,7 +972,7 @@ void FuncExit(ThreadState *thr) {
   }
 
   DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 #endif
   thr->shadow_stack_pos--;
@@ -966,7 +983,7 @@ void ThreadIgnoreBegin(ThreadState *thr, uptr pc) {
   thr->ignore_reads_and_writes++;
   CHECK_GT(thr->ignore_reads_and_writes, 0);
   thr->fast_state.SetIgnoreBit();
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   if (!ctx->after_multithreaded_fork)
     thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
 #endif
@@ -978,17 +995,25 @@ void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
   CHECK_GE(thr->ignore_reads_and_writes, 0);
   if (thr->ignore_reads_and_writes == 0) {
     thr->fast_state.ClearIgnoreBit();
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
     thr->mop_ignore_set.Reset();
 #endif
   }
 }
 
+#if !SANITIZER_GO
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+uptr __tsan_testonly_shadow_stack_current_size() {
+  ThreadState *thr = cur_thread();
+  return thr->shadow_stack_pos - thr->shadow_stack;
+}
+#endif
+
 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
   thr->ignore_sync++;
   CHECK_GT(thr->ignore_sync, 0);
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   if (!ctx->after_multithreaded_fork)
     thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
 #endif
@@ -998,7 +1023,7 @@ void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
   thr->ignore_sync--;
   CHECK_GE(thr->ignore_sync, 0);
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
   if (thr->ignore_sync == 0)
     thr->sync_ignore_set.Reset();
 #endif
@@ -1022,7 +1047,7 @@ void build_consistency_nostats() {}
 
 }  // namespace __tsan
 
-#ifndef SANITIZER_GO
+#if !SANITIZER_GO
 // Must be included in this file to make sure everything is inlined.
 #include "tsan_interface_inl.h"
 #endif