summaryrefslogtreecommitdiff
path: root/lib/tsan/rtl/tsan_rtl.h
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tsan/rtl/tsan_rtl.h')
-rw-r--r--lib/tsan/rtl/tsan_rtl.h322
1 files changed, 143 insertions, 179 deletions
diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h
index 4ee667543a6ea..8d886875159b3 100644
--- a/lib/tsan/rtl/tsan_rtl.h
+++ b/lib/tsan/rtl/tsan_rtl.h
@@ -28,7 +28,9 @@
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_asm.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
#include "sanitizer_common/sanitizer_libignore.h"
#include "sanitizer_common/sanitizer_suppressions.h"
#include "sanitizer_common/sanitizer_thread_registry.h"
@@ -41,6 +43,8 @@
#include "tsan_report.h"
#include "tsan_platform.h"
#include "tsan_mutexset.h"
+#include "tsan_ignoreset.h"
+#include "tsan_stack_trace.h"
#if SANITIZER_WORDSIZE != 64
# error "ThreadSanitizer is supported only on 64-bit platforms"
@@ -48,87 +52,9 @@
namespace __tsan {
-// Descriptor of user's memory block.
-struct MBlock {
- /*
- u64 mtx : 1; // must be first
- u64 lst : 44;
- u64 stk : 31; // on word boundary
- u64 tid : kTidBits;
- u64 siz : 128 - 1 - 31 - 44 - kTidBits; // 39
- */
- u64 raw[2];
-
- void Init(uptr siz, u32 tid, u32 stk) {
- raw[0] = raw[1] = 0;
- raw[1] |= (u64)siz << ((1 + 44 + 31 + kTidBits) % 64);
- raw[1] |= (u64)tid << ((1 + 44 + 31) % 64);
- raw[0] |= (u64)stk << (1 + 44);
- raw[1] |= (u64)stk >> (64 - 44 - 1);
- DCHECK_EQ(Size(), siz);
- DCHECK_EQ(Tid(), tid);
- DCHECK_EQ(StackId(), stk);
- }
-
- u32 Tid() const {
- return GetLsb(raw[1] >> ((1 + 44 + 31) % 64), kTidBits);
- }
-
- uptr Size() const {
- return raw[1] >> ((1 + 31 + 44 + kTidBits) % 64);
- }
-
- u32 StackId() const {
- return (raw[0] >> (1 + 44)) | GetLsb(raw[1] << (64 - 44 - 1), 31);
- }
-
- SyncVar *ListHead() const {
- return (SyncVar*)(GetLsb(raw[0] >> 1, 44) << 3);
- }
-
- void ListPush(SyncVar *v) {
- SyncVar *lst = ListHead();
- v->next = lst;
- u64 x = (u64)v ^ (u64)lst;
- x = (x >> 3) << 1;
- raw[0] ^= x;
- DCHECK_EQ(ListHead(), v);
- }
-
- SyncVar *ListPop() {
- SyncVar *lst = ListHead();
- SyncVar *nxt = lst->next;
- lst->next = 0;
- u64 x = (u64)lst ^ (u64)nxt;
- x = (x >> 3) << 1;
- raw[0] ^= x;
- DCHECK_EQ(ListHead(), nxt);
- return lst;
- }
-
- void ListReset() {
- SyncVar *lst = ListHead();
- u64 x = (u64)lst;
- x = (x >> 3) << 1;
- raw[0] ^= x;
- DCHECK_EQ(ListHead(), 0);
- }
-
- void Lock();
- void Unlock();
- typedef GenericScopedLock<MBlock> ScopedLock;
-};
-
-#ifndef TSAN_GO
-#if defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW
-const uptr kAllocatorSpace = 0x7d0000000000ULL;
-#else
-const uptr kAllocatorSpace = 0x7d0000000000ULL;
-#endif
-const uptr kAllocatorSize = 0x10000000000ULL; // 1T.
-
+#ifndef SANITIZER_GO
struct MapUnmapCallback;
-typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, sizeof(MBlock),
+typedef SizeClassAllocator64<kHeapMemBeg, kHeapMemEnd - kHeapMemBeg, 0,
DefaultSizeClassMap, MapUnmapCallback> PrimaryAllocator;
typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator;
@@ -145,14 +71,14 @@ const u64 kShadowRodata = (u64)-1; // .rodata shadow marker
// FastState (from most significant bit):
// ignore : 1
// tid : kTidBits
-// epoch : kClkBits
// unused : -
// history_size : 3
+// epoch : kClkBits
class FastState {
public:
FastState(u64 tid, u64 epoch) {
x_ = tid << kTidShift;
- x_ |= epoch << kClkShift;
+ x_ |= epoch;
DCHECK_EQ(tid, this->tid());
DCHECK_EQ(epoch, this->epoch());
DCHECK_EQ(GetIgnoreBit(), false);
@@ -177,13 +103,13 @@ class FastState {
}
u64 epoch() const {
- u64 res = (x_ << (kTidBits + 1)) >> (64 - kClkBits);
+ u64 res = x_ & ((1ull << kClkBits) - 1);
return res;
}
void IncrementEpoch() {
u64 old_epoch = epoch();
- x_ += 1 << kClkShift;
+ x_ += 1;
DCHECK_EQ(old_epoch + 1, epoch());
(void)old_epoch;
}
@@ -195,17 +121,19 @@ class FastState {
void SetHistorySize(int hs) {
CHECK_GE(hs, 0);
CHECK_LE(hs, 7);
- x_ = (x_ & ~7) | hs;
+ x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift);
}
+ ALWAYS_INLINE
int GetHistorySize() const {
- return (int)(x_ & 7);
+ return (int)((x_ >> kHistoryShift) & kHistoryMask);
}
void ClearHistorySize() {
- x_ &= ~7;
+ SetHistorySize(0);
}
+ ALWAYS_INLINE
u64 GetTracePos() const {
const int hs = GetHistorySize();
// When hs == 0, the trace consists of 2 parts.
@@ -216,20 +144,21 @@ class FastState {
private:
friend class Shadow;
static const int kTidShift = 64 - kTidBits - 1;
- static const int kClkShift = kTidShift - kClkBits;
static const u64 kIgnoreBit = 1ull << 63;
static const u64 kFreedBit = 1ull << 63;
+ static const u64 kHistoryShift = kClkBits;
+ static const u64 kHistoryMask = 7;
u64 x_;
};
// Shadow (from most significant bit):
// freed : 1
// tid : kTidBits
-// epoch : kClkBits
// is_atomic : 1
// is_read : 1
// size_log : 2
// addr0 : 3
+// epoch : kClkBits
class Shadow : public FastState {
public:
explicit Shadow(u64 x)
@@ -242,10 +171,10 @@ class Shadow : public FastState {
}
void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
- DCHECK_EQ(x_ & 31, 0);
+ DCHECK_EQ((x_ >> kClkBits) & 31, 0);
DCHECK_LE(addr0, 7);
DCHECK_LE(kAccessSizeLog, 3);
- x_ |= (kAccessSizeLog << 3) | addr0;
+ x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits;
DCHECK_EQ(kAccessSizeLog, size_log());
DCHECK_EQ(addr0, this->addr0());
}
@@ -278,47 +207,34 @@ class Shadow : public FastState {
return shifted_xor == 0;
}
- static inline bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
- u64 masked_xor = (s1.x_ ^ s2.x_) & 31;
+ static ALWAYS_INLINE
+ bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
+ u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
return masked_xor == 0;
}
- static inline bool TwoRangesIntersect(Shadow s1, Shadow s2,
+ static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
unsigned kS2AccessSize) {
bool res = false;
u64 diff = s1.addr0() - s2.addr0();
if ((s64)diff < 0) { // s1.addr0 < s2.addr0 // NOLINT
// if (s1.addr0() + size1) > s2.addr0()) return true;
- if (s1.size() > -diff) res = true;
+ if (s1.size() > -diff)
+ res = true;
} else {
// if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
- if (kS2AccessSize > diff) res = true;
+ if (kS2AccessSize > diff)
+ res = true;
}
- DCHECK_EQ(res, TwoRangesIntersectSLOW(s1, s2));
- DCHECK_EQ(res, TwoRangesIntersectSLOW(s2, s1));
+ DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
+ DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
return res;
}
- // The idea behind the offset is as follows.
- // Consider that we have 8 bool's contained within a single 8-byte block
- // (mapped to a single shadow "cell"). Now consider that we write to the bools
- // from a single thread (which we consider the common case).
- // W/o offsetting each access will have to scan 4 shadow values at average
- // to find the corresponding shadow value for the bool.
- // With offsetting we start scanning shadow with the offset so that
- // each access hits necessary shadow straight off (at least in an expected
- // optimistic case).
- // This logic works seamlessly for any layout of user data. For example,
- // if user data is {int, short, char, char}, then accesses to the int are
- // offsetted to 0, short - 4, 1st char - 6, 2nd char - 7. Hopefully, accesses
- // from a single thread won't need to scan all 8 shadow values.
- unsigned ComputeSearchOffset() {
- return x_ & 7;
- }
- u64 addr0() const { return x_ & 7; }
- u64 size() const { return 1ull << size_log(); }
- bool IsWrite() const { return !IsRead(); }
- bool IsRead() const { return x_ & kReadBit; }
+ u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
+ u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
+ bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
+ bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
// The idea behind the freed bit is as follows.
// When the memory is freed (or otherwise unaccessible) we write to the shadow
@@ -343,15 +259,14 @@ class Shadow : public FastState {
return res;
}
- bool IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
- // analyzes 5-th bit (is_read) and 6-th bit (is_atomic)
- bool v = x_ & u64(((kIsWrite ^ 1) << kReadShift)
- | (kIsAtomic << kAtomicShift));
+ bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
+ bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift)
+ | (u64(kIsAtomic) << kAtomicShift));
DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic));
return v;
}
- bool IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
+ bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
bool v = ((x_ >> kReadShift) & 3)
<= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
DCHECK_EQ(v, (IsAtomic() < kIsAtomic) ||
@@ -359,7 +274,7 @@ class Shadow : public FastState {
return v;
}
- bool IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
+ bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
bool v = ((x_ >> kReadShift) & 3)
>= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
DCHECK_EQ(v, (IsAtomic() > kIsAtomic) ||
@@ -368,14 +283,14 @@ class Shadow : public FastState {
}
private:
- static const u64 kReadShift = 5;
+ static const u64 kReadShift = 5 + kClkBits;
static const u64 kReadBit = 1ull << kReadShift;
- static const u64 kAtomicShift = 6;
+ static const u64 kAtomicShift = 6 + kClkBits;
static const u64 kAtomicBit = 1ull << kAtomicShift;
- u64 size_log() const { return (x_ >> 3) & 3; }
+ u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
- static bool TwoRangesIntersectSLOW(const Shadow s1, const Shadow s2) {
+ static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
if (s1.addr0() == s2.addr0()) return true;
if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
return true;
@@ -390,6 +305,9 @@ struct SignalContext;
struct JmpBuf {
uptr sp;
uptr mangled_sp;
+ int int_signal_send;
+ bool in_blocking_func;
+ uptr in_signal_handler;
uptr *shadow_stack_pos;
};
@@ -413,6 +331,11 @@ struct ThreadState {
// for better performance.
int ignore_reads_and_writes;
int ignore_sync;
+ // Go does not support ignores.
+#ifndef SANITIZER_GO
+ IgnoreSet mop_ignore_set;
+ IgnoreSet sync_ignore_set;
+#endif
// C/C++ uses fixed size shadow stack embed into Trace.
// Go uses malloc-allocated shadow stack with dynamic size.
uptr *shadow_stack;
@@ -422,31 +345,38 @@ struct ThreadState {
u64 racy_state[2];
MutexSet mset;
ThreadClock clock;
-#ifndef TSAN_GO
+#ifndef SANITIZER_GO
AllocatorCache alloc_cache;
InternalAllocatorCache internal_alloc_cache;
Vector<JmpBuf> jmp_bufs;
+ int ignore_interceptors;
#endif
u64 stat[StatCnt];
const int tid;
const int unique_id;
- int in_rtl;
bool in_symbolizer;
bool in_ignored_lib;
- bool is_alive;
+ bool is_dead;
bool is_freeing;
bool is_vptr_access;
const uptr stk_addr;
const uptr stk_size;
const uptr tls_addr;
const uptr tls_size;
+ ThreadContext *tctx;
- DeadlockDetector deadlock_detector;
+ InternalDeadlockDetector internal_deadlock_detector;
+ DDPhysicalThread *dd_pt;
+ DDLogicalThread *dd_lt;
- bool in_signal_handler;
+ atomic_uintptr_t in_signal_handler;
SignalContext *signal_ctx;
-#ifndef TSAN_GO
+ DenseSlabAllocCache block_cache;
+ DenseSlabAllocCache sync_cache;
+ DenseSlabAllocCache clock_cache;
+
+#ifndef SANITIZER_GO
u32 last_sleep_stack_id;
ThreadClock last_sleep_clock;
#endif
@@ -456,13 +386,13 @@ struct ThreadState {
int nomalloc;
explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
+ unsigned reuse_count,
uptr stk_addr, uptr stk_size,
uptr tls_addr, uptr tls_size);
};
-Context *CTX();
-
-#ifndef TSAN_GO
+#ifndef SANITIZER_GO
+__attribute__((tls_model("initial-exec")))
extern THREADLOCAL char cur_thread_placeholder[];
INLINE ThreadState *cur_thread() {
return reinterpret_cast<ThreadState *>(&cur_thread_placeholder);
@@ -474,11 +404,7 @@ class ThreadContext : public ThreadContextBase {
explicit ThreadContext(int tid);
~ThreadContext();
ThreadState *thr;
-#ifdef TSAN_GO
- StackTrace creation_stack;
-#else
u32 creation_stack_id;
-#endif
SyncClock sync;
// Epoch at which the thread had started.
// If we see an event from the thread stamped by an older epoch,
@@ -493,6 +419,7 @@ class ThreadContext : public ThreadContextBase {
void OnStarted(void *arg);
void OnCreated(void *arg);
void OnReset();
+ void OnDetached(void *arg);
};
struct RacyStacks {
@@ -521,20 +448,27 @@ struct Context {
Context();
bool initialized;
+ bool after_multithreaded_fork;
- SyncTab synctab;
+ MetaMap metamap;
Mutex report_mtx;
int nreported;
int nmissed_expected;
atomic_uint64_t last_symbolize_time_ns;
+ void *background_thread;
+ atomic_uint32_t stop_background_thread;
+
ThreadRegistry *thread_registry;
Vector<RacyStacks> racy_stacks;
Vector<RacyAddress> racy_addresses;
// Number of fired suppressions may be large enough.
InternalMmapVector<FiredSuppression> fired_suppressions;
+ DDetector *dd;
+
+ ClockAlloc clock_alloc;
Flags flags;
@@ -543,14 +477,20 @@ struct Context {
u64 int_alloc_siz[MBlockTypeCount];
};
-class ScopedInRtl {
- public:
- ScopedInRtl();
- ~ScopedInRtl();
- private:
- ThreadState*thr_;
- int in_rtl_;
- int errno_;
+extern Context *ctx; // The one and the only global runtime context.
+
+struct ScopedIgnoreInterceptors {
+ ScopedIgnoreInterceptors() {
+#ifndef SANITIZER_GO
+ cur_thread()->ignore_interceptors++;
+#endif
+ }
+
+ ~ScopedIgnoreInterceptors() {
+#ifndef SANITIZER_GO
+ cur_thread()->ignore_interceptors--;
+#endif
+ }
};
class ScopedReport {
@@ -558,11 +498,14 @@ class ScopedReport {
explicit ScopedReport(ReportType typ);
~ScopedReport();
- void AddStack(const StackTrace *stack);
- void AddMemoryAccess(uptr addr, Shadow s, const StackTrace *stack,
+ void AddMemoryAccess(uptr addr, Shadow s, StackTrace stack,
const MutexSet *mset);
- void AddThread(const ThreadContext *tctx);
+ void AddStack(StackTrace stack, bool suppressable = false);
+ void AddThread(const ThreadContext *tctx, bool suppressable = false);
+ void AddThread(int unique_tid, bool suppressable = false);
+ void AddUniqueTid(int unique_tid);
void AddMutex(const SyncVar *s);
+ u64 AddMutex(u64 id);
void AddLocation(uptr addr, uptr size);
void AddSleep(u32 stack_id);
void SetCount(int count);
@@ -570,16 +513,31 @@ class ScopedReport {
const ReportDesc *GetReport() const;
private:
- Context *ctx_;
ReportDesc *rep_;
+ // Symbolizer makes lots of intercepted calls. If we try to process them,
+ // at best it will cause deadlocks on internal mutexes.
+ ScopedIgnoreInterceptors ignore_interceptors_;
- void AddMutex(u64 id);
+ void AddDeadMutex(u64 id);
ScopedReport(const ScopedReport&);
void operator = (const ScopedReport&);
};
-void RestoreStack(int tid, const u64 epoch, StackTrace *stk, MutexSet *mset);
+void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
+ MutexSet *mset);
+
+template<typename StackTraceTy>
+void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) {
+ uptr size = thr->shadow_stack_pos - thr->shadow_stack;
+ uptr start = 0;
+ if (size + !!toppc > kStackTraceMax) {
+ start = size + !!toppc - kStackTraceMax;
+ size = kStackTraceMax - !!toppc;
+ }
+ stack->Init(&thr->shadow_stack[start], size, toppc);
+}
+
void StatAggregate(u64 *dst, u64 *src);
void StatOutput(u64 *stat);
@@ -600,19 +558,16 @@ void InitializeInterceptors();
void InitializeLibIgnore();
void InitializeDynamicAnnotations();
+void ForkBefore(ThreadState *thr, uptr pc);
+void ForkParentAfter(ThreadState *thr, uptr pc);
+void ForkChildAfter(ThreadState *thr, uptr pc);
+
void ReportRace(ThreadState *thr);
-bool OutputReport(Context *ctx,
- const ScopedReport &srep,
- const ReportStack *suppress_stack1 = 0,
- const ReportStack *suppress_stack2 = 0,
- const ReportLocation *suppress_loc = 0);
-bool IsFiredSuppression(Context *ctx,
- const ScopedReport &srep,
- const StackTrace &trace);
+bool OutputReport(ThreadState *thr, const ScopedReport &srep);
+bool IsFiredSuppression(Context *ctx, const ScopedReport &srep,
+ StackTrace trace);
bool IsExpectedReport(uptr addr, uptr size);
void PrintMatchedBenignRaces();
-bool FrameIsInternal(const ReportStack *frame);
-ReportStack *SkipTsanInternalFrames(ReportStack *ent);
#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
# define DPrintf Printf
@@ -627,15 +582,15 @@ ReportStack *SkipTsanInternalFrames(ReportStack *ent);
#endif
u32 CurrentStackId(ThreadState *thr, uptr pc);
+ReportStack *SymbolizeStackId(u32 stack_id);
void PrintCurrentStack(ThreadState *thr, uptr pc);
-void PrintCurrentStackSlow(); // uses libunwind
+void PrintCurrentStackSlow(uptr pc); // uses libunwind
void Initialize(ThreadState *thr);
int Finalize(ThreadState *thr);
-SyncVar* GetJavaSync(ThreadState *thr, uptr pc, uptr addr,
- bool write_lock, bool create);
-SyncVar* GetAndRemoveJavaSync(ThreadState *thr, uptr pc, uptr addr);
+void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
+void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
@@ -678,10 +633,10 @@ void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
-void ThreadIgnoreBegin(ThreadState *thr);
-void ThreadIgnoreEnd(ThreadState *thr);
-void ThreadIgnoreSyncBegin(ThreadState *thr);
-void ThreadIgnoreSyncEnd(ThreadState *thr);
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
void FuncEntry(ThreadState *thr, uptr pc);
void FuncExit(ThreadState *thr);
@@ -700,14 +655,21 @@ void ProcessPendingSignals(ThreadState *thr);
void MutexCreate(ThreadState *thr, uptr pc, uptr addr,
bool rw, bool recursive, bool linker_init);
void MutexDestroy(ThreadState *thr, uptr pc, uptr addr);
-void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1);
+void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1,
+ bool try_lock = false);
int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all = false);
-void MutexReadLock(ThreadState *thr, uptr pc, uptr addr);
+void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool try_lock = false);
void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD
void Acquire(ThreadState *thr, uptr pc, uptr addr);
+// AcquireGlobal synchronizes the current thread with all other threads.
+// In terms of happens-before relation, it draws a HB edge from all threads
+// (where they happen to execute right now) to the current thread. We use it to
+// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
+// right before executing finalizers. This provides a coarse, but simple
+// approximation of the actual required synchronization.
void AcquireGlobal(ThreadState *thr, uptr pc);
void Release(ThreadState *thr, uptr pc, uptr addr);
void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
@@ -728,11 +690,11 @@ void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
// so we create a reserve stack frame for it (1024b must be enough).
#define HACKY_CALL(f) \
__asm__ __volatile__("sub $1024, %%rsp;" \
- ".cfi_adjust_cfa_offset 1024;" \
+ CFI_INL_ADJUST_CFA_OFFSET(1024) \
".hidden " #f "_thunk;" \
"call " #f "_thunk;" \
"add $1024, %%rsp;" \
- ".cfi_adjust_cfa_offset -1024;" \
+ CFI_INL_ADJUST_CFA_OFFSET(-1024) \
::: "memory", "cc");
#else
#define HACKY_CALL(f) f()
@@ -747,13 +709,15 @@ Trace *ThreadTrace(int tid);
extern "C" void __tsan_trace_switch();
void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
EventType typ, u64 addr) {
+ if (!kCollectHistory)
+ return;
DCHECK_GE((int)typ, 0);
DCHECK_LE((int)typ, 7);
DCHECK_EQ(GetLsb(addr, 61), addr);
StatInc(thr, StatEvents);
u64 pos = fs.GetTracePos();
if (UNLIKELY((pos % kTracePartSize) == 0)) {
-#ifndef TSAN_GO
+#ifndef SANITIZER_GO
HACKY_CALL(__tsan_trace_switch);
#else
TraceSwitch(thr);
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-28 06:25:13 +0000