diff options
Diffstat (limited to 'lib/sanitizer_common/sanitizer_coverage_libcdep.cc')
| -rw-r--r-- | lib/sanitizer_common/sanitizer_coverage_libcdep.cc | 345 |
1 files changed, 241 insertions, 104 deletions
diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc index 3fa8a18696f8f..e8f42f68a42ff 100644 --- a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc @@ -12,14 +12,16 @@ // // Compiler instrumentation: // For every interesting basic block the compiler injects the following code: -// if (Guard) { +// if (Guard < 0) { // __sanitizer_cov(&Guard); // } +// At the module start up time __sanitizer_cov_module_init sets the guards +// to consecutive negative numbers (-1, -2, -3, ...). // It's fine to call __sanitizer_cov more than once for a given block. // // Run-time: // - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC). -// and atomically set Guard to 1. +// and atomically set Guard to -Guard. // - __sanitizer_cov_dump: dump the coverage data to disk. // For every module of the current process that has coverage data // this will create a file module_name.PID.sancov. The file format is simple: @@ -56,23 +58,32 @@ static atomic_uintptr_t coverage_counter; static bool cov_sandboxed = false; static int cov_fd = kInvalidFd; static unsigned int cov_max_block_size = 0; +static bool coverage_enabled = false; +static const char *coverage_dir; namespace __sanitizer { class CoverageData { public: void Init(); + void Enable(); + void Disable(); + void ReInit(); void BeforeFork(); void AfterFork(int child_pid); void Extend(uptr npcs); - void Add(uptr pc, u8 *guard); + void Add(uptr pc, u32 *guard); void IndirCall(uptr caller, uptr callee, uptr callee_cache[], uptr cache_size); void DumpCallerCalleePairs(); void DumpTrace(); ALWAYS_INLINE - void TraceBasicaBlock(uptr *cache); + void TraceBasicBlock(s32 *id); + + void InitializeGuardArray(s32 *guards); + void InitializeGuards(s32 *guards, uptr n); + void ReinitializeGuards(); uptr *data(); uptr size(); @@ -80,7 +91,7 @@ class CoverageData { private: // Maximal size pc array may ever grow. // We MmapNoReserve this space to ensure that the array is contiguous. - static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); + static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 24, 1 << 27); // The amount file mapping for the pc array is grown by. static const uptr kPcArrayMmapSize = 64 * 1024; @@ -96,45 +107,41 @@ class CoverageData { // Descriptor of the file mapped pc array. int pc_fd; + // Vector of coverage guard arrays, protected by mu. + InternalMmapVectorNoCtor<s32*> guard_array_vec; + // Caller-Callee (cc) array, size and current index. static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24); uptr **cc_array; atomic_uintptr_t cc_array_index; atomic_uintptr_t cc_array_size; - // Tracing (tr) pc and event arrays, their size and current index. + // Tracing event array, size and current pointer. // We record all events (basic block entries) in a global buffer of u32 - // values. Each such value is an index in the table of TracedPc objects. + // values. Each such value is the index in pc_array. // So far the tracing is highly experimental: // - not thread-safe; // - does not support long traces; // - not tuned for performance. - struct TracedPc { - uptr pc; - const char *module_name; - uptr module_offset; - }; static const uptr kTrEventArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 30); u32 *tr_event_array; uptr tr_event_array_size; - uptr tr_event_array_index; + u32 *tr_event_pointer; static const uptr kTrPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); - TracedPc *tr_pc_array; - uptr tr_pc_array_size; - uptr tr_pc_array_index; StaticSpinMutex mu; void DirectOpen(); - void ReInit(); }; static CoverageData coverage_data; +void CovUpdateMapping(const char *path, uptr caller_pc = 0); + void CoverageData::DirectOpen() { InternalScopedString path(kMaxPathLength); internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw", - common_flags()->coverage_dir, internal_getpid()); + coverage_dir, internal_getpid()); pc_fd = OpenFile(path.data(), true); if (internal_iserror(pc_fd)) { Report(" Coverage: failed to open %s for writing\n", path.data()); @@ -142,19 +149,23 @@ void CoverageData::DirectOpen() { } pc_array_mapped_size = 0; - CovUpdateMapping(); + CovUpdateMapping(coverage_dir); } void CoverageData::Init() { + pc_fd = kInvalidFd; +} + +void CoverageData::Enable() { + if (pc_array) + return; pc_array = reinterpret_cast<uptr *>( MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit")); - pc_fd = kInvalidFd; + atomic_store(&pc_array_index, 0, memory_order_relaxed); if (common_flags()->coverage_direct) { atomic_store(&pc_array_size, 0, memory_order_relaxed); - atomic_store(&pc_array_index, 0, memory_order_relaxed); } else { atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed); - atomic_store(&pc_array_index, 0, memory_order_relaxed); } cc_array = reinterpret_cast<uptr **>(MmapNoReserveOrDie( @@ -162,30 +173,72 @@ void CoverageData::Init() { atomic_store(&cc_array_size, kCcArrayMaxSize, memory_order_relaxed); atomic_store(&cc_array_index, 0, memory_order_relaxed); - tr_event_array = reinterpret_cast<u32 *>( - MmapNoReserveOrDie(sizeof(tr_event_array[0]) * kTrEventArrayMaxSize, - "CovInit::tr_event_array")); + // Allocate tr_event_array with a guard page at the end. + tr_event_array = reinterpret_cast<u32 *>(MmapNoReserveOrDie( + sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(), + "CovInit::tr_event_array")); + Mprotect(reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]), + GetMmapGranularity()); tr_event_array_size = kTrEventArrayMaxSize; - tr_event_array_index = 0; + tr_event_pointer = tr_event_array; +} - tr_pc_array = reinterpret_cast<TracedPc *>(MmapNoReserveOrDie( - sizeof(tr_pc_array[0]) * kTrEventArrayMaxSize, "CovInit::tr_pc_array")); - tr_pc_array_size = kTrEventArrayMaxSize; - tr_pc_array_index = 0; +void CoverageData::InitializeGuardArray(s32 *guards) { + Enable(); // Make sure coverage is enabled at this point. + s32 n = guards[0]; + for (s32 j = 1; j <= n; j++) { + uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed); + guards[j] = -static_cast<s32>(idx + 1); + } +} + +void CoverageData::Disable() { + if (pc_array) { + internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize); + pc_array = nullptr; + } + if (cc_array) { + internal_munmap(cc_array, sizeof(uptr *) * kCcArrayMaxSize); + cc_array = nullptr; + } + if (tr_event_array) { + internal_munmap(tr_event_array, + sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + + GetMmapGranularity()); + tr_event_array = nullptr; + tr_event_pointer = nullptr; + } + if (pc_fd != kInvalidFd) { + internal_close(pc_fd); + pc_fd = kInvalidFd; + } +} + +void CoverageData::ReinitializeGuards() { + // Assuming single thread. + atomic_store(&pc_array_index, 0, memory_order_relaxed); + for (uptr i = 0; i < guard_array_vec.size(); i++) + InitializeGuardArray(guard_array_vec[i]); } void CoverageData::ReInit() { - internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize); - if (pc_fd != kInvalidFd) internal_close(pc_fd); - if (common_flags()->coverage_direct) { - // In memory-mapped mode we must extend the new file to the known array - // size. - uptr size = atomic_load(&pc_array_size, memory_order_relaxed); - Init(); - if (size) Extend(size); - } else { - Init(); + Disable(); + if (coverage_enabled) { + if (common_flags()->coverage_direct) { + // In memory-mapped mode we must extend the new file to the known array + // size. + uptr size = atomic_load(&pc_array_size, memory_order_relaxed); + Enable(); + if (size) Extend(size); + if (coverage_enabled) CovUpdateMapping(coverage_dir); + } else { + Enable(); + } } + // Re-initialize the guards. + // We are single-threaded now, no need to grab any lock. + CHECK_EQ(atomic_load(&pc_array_index, memory_order_relaxed), 0); + ReinitializeGuards(); } void CoverageData::BeforeFork() { @@ -203,15 +256,16 @@ void CoverageData::Extend(uptr npcs) { if (!common_flags()->coverage_direct) return; SpinMutexLock l(&mu); - if (pc_fd == kInvalidFd) DirectOpen(); - CHECK_NE(pc_fd, kInvalidFd); - uptr size = atomic_load(&pc_array_size, memory_order_relaxed); size += npcs * sizeof(uptr); - if (size > pc_array_mapped_size) { + if (coverage_enabled && size > pc_array_mapped_size) { + if (pc_fd == kInvalidFd) DirectOpen(); + CHECK_NE(pc_fd, kInvalidFd); + uptr new_mapped_size = pc_array_mapped_size; while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize; + CHECK_LE(new_mapped_size, sizeof(uptr) * kPcArrayMaxSize); // Extend the file and map the new space at the end of pc_array. uptr res = internal_ftruncate(pc_fd, new_mapped_size); @@ -220,29 +274,45 @@ void CoverageData::Extend(uptr npcs) { Printf("failed to extend raw coverage file: %d\n", err); Die(); } - void *p = MapWritableFileToMemory(pc_array + pc_array_mapped_size, + + uptr next_map_base = ((uptr)pc_array) + pc_array_mapped_size; + void *p = MapWritableFileToMemory((void *)next_map_base, new_mapped_size - pc_array_mapped_size, pc_fd, pc_array_mapped_size); - CHECK_EQ(p, pc_array + pc_array_mapped_size); + CHECK_EQ((uptr)p, next_map_base); pc_array_mapped_size = new_mapped_size; } atomic_store(&pc_array_size, size, memory_order_release); } -// Atomically add the pc to the vector. The atomically set the guard to 1. -// If the function is called more than once for a given PC it will -// be inserted multiple times, which is fine. -void CoverageData::Add(uptr pc, u8 *guard) { +void CoverageData::InitializeGuards(s32 *guards, uptr n) { + // The array 'guards' has n+1 elements, we use the element zero + // to store 'n'. + CHECK_LT(n, 1 << 30); + guards[0] = static_cast<s32>(n); + InitializeGuardArray(guards); + SpinMutexLock l(&mu); + guard_array_vec.push_back(guards); +} + +// If guard is negative, atomically set it to -guard and store the PC in +// pc_array. +void CoverageData::Add(uptr pc, u32 *guard) { + atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard); + s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed); + if (guard_value >= 0) return; + + atomic_store(atomic_guard, -guard_value, memory_order_relaxed); if (!pc_array) return; - uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed); + + uptr idx = -guard_value - 1; + if (idx >= atomic_load(&pc_array_index, memory_order_acquire)) + return; // May happen after fork when pc_array_index becomes 0. CHECK_LT(idx * sizeof(uptr), atomic_load(&pc_array_size, memory_order_acquire)); pc_array[idx] = pc; atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); - // Set the guard. - atomic_uint8_t *atomic_guard = reinterpret_cast<atomic_uint8_t*>(guard); - atomic_store(atomic_guard, 1, memory_order_relaxed); } // Registers a pair caller=>callee. @@ -338,18 +408,19 @@ static void CovWritePacked(int pid, const char *module, const void *blob, // If packed = true and name == 0: <pid>.<sancov>.<packed>. // If packed = true and name != 0: <name>.<sancov>.<packed> (name is // user-supplied). -static int CovOpenFile(bool packed, const char* name) { +static int CovOpenFile(bool packed, const char *name, + const char *extension = "sancov") { InternalScopedString path(kMaxPathLength); if (!packed) { CHECK(name); - path.append("%s/%s.%zd.sancov", common_flags()->coverage_dir, name, - internal_getpid()); + path.append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(), + extension); } else { if (!name) - path.append("%s/%zd.sancov.packed", common_flags()->coverage_dir, - internal_getpid()); + path.append("%s/%zd.%s.packed", coverage_dir, internal_getpid(), + extension); else - path.append("%s/%s.sancov.packed", common_flags()->coverage_dir, name); + path.append("%s/%s.%s.packed", coverage_dir, name, extension); } uptr fd = OpenFile(path.data(), true); if (internal_iserror(fd)) { @@ -361,23 +432,18 @@ static int CovOpenFile(bool packed, const char* name) { // Dump trace PCs and trace events into two separate files. void CoverageData::DumpTrace() { - uptr max_idx = tr_event_array_index; + uptr max_idx = tr_event_pointer - tr_event_array; if (!max_idx) return; auto sym = Symbolizer::GetOrInit(); if (!sym) return; InternalScopedString out(32 << 20); - for (uptr i = 0; i < max_idx; i++) { - u32 pc_idx = tr_event_array[i]; - TracedPc *t = &tr_pc_array[pc_idx]; - if (!t->module_name) { - const char *module_name = "<unknown>"; - uptr module_address = 0; - sym->GetModuleNameAndOffsetForPC(t->pc, &module_name, &module_address); - t->module_name = internal_strdup(module_name); - t->module_offset = module_address; - out.append("%s 0x%zx\n", t->module_name, t->module_offset); - } + for (uptr i = 0, n = size(); i < n; i++) { + const char *module_name = "<unknown>"; + uptr module_address = 0; + sym->GetModuleNameAndOffsetForPC(pc_array[i], &module_name, + &module_address); + out.append("%s 0x%zx\n", module_name, module_address); } int fd = CovOpenFile(false, "trace-points"); if (fd < 0) return; @@ -386,10 +452,21 @@ void CoverageData::DumpTrace() { fd = CovOpenFile(false, "trace-events"); if (fd < 0) return; - internal_write(fd, tr_event_array, max_idx * sizeof(tr_event_array[0])); + uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]); + u8 *event_bytes = reinterpret_cast<u8*>(tr_event_array); + // The trace file could be huge, and may not be written with a single syscall. + while (bytes_to_write) { + uptr actually_written = internal_write(fd, event_bytes, bytes_to_write); + if (actually_written <= bytes_to_write) { + bytes_to_write -= actually_written; + event_bytes += actually_written; + } else { + break; + } + } internal_close(fd); - VReport(1, " CovDump: Trace: %zd PCs written\n", tr_pc_array_index); - VReport(1, " CovDump: Trace: %zd Events written\n", tr_event_array_index); + VReport(1, " CovDump: Trace: %zd PCs written\n", size()); + VReport(1, " CovDump: Trace: %zd Events written\n", max_idx); } // This function dumps the caller=>callee pairs into a file as a sequence of @@ -434,28 +511,45 @@ void CoverageData::DumpCallerCalleePairs() { // Record the current PC into the event buffer. // Every event is a u32 value (index in tr_pc_array_index) so we compute // it once and then cache in the provided 'cache' storage. -void CoverageData::TraceBasicaBlock(uptr *cache) { - CHECK(common_flags()->coverage); - uptr idx = *cache; - if (!idx) { - CHECK_LT(tr_pc_array_index, kTrPcArrayMaxSize); - idx = tr_pc_array_index++; - TracedPc *t = &tr_pc_array[idx]; - t->pc = GET_CALLER_PC(); - *cache = idx; - CHECK_LT(idx, 1U << 31); +// +// This function will eventually be inlined by the compiler. +void CoverageData::TraceBasicBlock(s32 *id) { + // Will trap here if + // 1. coverage is not enabled at run-time. + // 2. The array tr_event_array is full. + *tr_event_pointer = static_cast<u32>(*id - 1); + tr_event_pointer++; +} + +static void CovDumpAsBitSet() { + if (!common_flags()->coverage_bitset) return; + if (!coverage_data.size()) return; + int fd = CovOpenFile(/* packed */false, "combined", "bitset-sancov"); + if (fd < 0) return; + uptr n = coverage_data.size(); + uptr n_set_bits = 0; + InternalScopedBuffer<char> out(n); + for (uptr i = 0; i < n; i++) { + uptr pc = coverage_data.data()[i]; + out[i] = pc ? '1' : '0'; + if (pc) + n_set_bits++; } - CHECK_LT(tr_event_array_index, tr_event_array_size); - tr_event_array[tr_event_array_index] = static_cast<u32>(idx); - tr_event_array_index++; + internal_write(fd, out.data(), n); + internal_close(fd); + VReport(1, " CovDump: bitset of %zd bits written, %zd bits are set\n", n, + n_set_bits); } // Dump the coverage on disk. static void CovDump() { - if (!common_flags()->coverage || common_flags()->coverage_direct) return; + if (!coverage_enabled || common_flags()->coverage_direct) return; #if !SANITIZER_WINDOWS if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed)) return; + CovDumpAsBitSet(); + coverage_data.DumpTrace(); + if (!common_flags()->coverage_pcs) return; uptr size = coverage_data.size(); InternalMmapVector<u32> offsets(size); uptr *vb = coverage_data.data(); @@ -491,8 +585,8 @@ static void CovDump() { } else { // One file per module per process. path.clear(); - path.append("%s/%s.%zd.sancov", common_flags()->coverage_dir, - module_name, internal_getpid()); + path.append("%s/%s.%zd.sancov", coverage_dir, module_name, + internal_getpid()); int fd = CovOpenFile(false /* packed */, module_name); if (fd > 0) { internal_write(fd, offsets.data(), offsets.size() * sizeof(u32)); @@ -506,13 +600,12 @@ static void CovDump() { if (cov_fd >= 0) internal_close(cov_fd); coverage_data.DumpCallerCalleePairs(); - coverage_data.DumpTrace(); #endif // !SANITIZER_WINDOWS } void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { if (!args) return; - if (!common_flags()->coverage) return; + if (!coverage_enabled) return; cov_sandboxed = args->coverage_sandboxed; if (!cov_sandboxed) return; cov_fd = args->coverage_fd; @@ -524,7 +617,7 @@ void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { int MaybeOpenCovFile(const char *name) { CHECK(name); - if (!common_flags()->coverage) return -1; + if (!coverage_enabled) return -1; return CovOpenFile(true /* packed */, name); } @@ -536,28 +629,60 @@ void CovAfterFork(int child_pid) { coverage_data.AfterFork(child_pid); } +void InitializeCoverage(bool enabled, const char *dir) { + if (coverage_enabled) + return; // May happen if two sanitizer enable coverage in the same process. + coverage_enabled = enabled; + coverage_dir = dir; + coverage_data.Init(); + if (enabled) coverage_data.Enable(); +#if !SANITIZER_WINDOWS + if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump); +#endif +} + +void ReInitializeCoverage(bool enabled, const char *dir) { + coverage_enabled = enabled; + coverage_dir = dir; + coverage_data.ReInit(); +} + +void CoverageUpdateMapping() { + if (coverage_enabled) + CovUpdateMapping(coverage_dir); +} + } // namespace __sanitizer extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u8 *guard) { +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) { coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), guard); } +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) { + atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard); + if (__sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) + __sanitizer_cov(guard); +} SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_indir_call16(uptr callee, uptr callee_cache16[]) { coverage_data.IndirCall(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), callee, callee_cache16, 16); } -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { + coverage_enabled = true; + coverage_dir = common_flags()->coverage_dir; coverage_data.Init(); } -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(uptr npcs) { - if (!common_flags()->coverage || !common_flags()->coverage_direct) return; - if (SANITIZER_ANDROID) { +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); } +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(s32 *guards, + uptr npcs) { + coverage_data.InitializeGuards(guards, npcs); + if (!common_flags()->coverage_direct) return; + if (SANITIZER_ANDROID && coverage_enabled) { // dlopen/dlclose interceptors do not work on Android, so we rely on // Extend() calls to update .sancov.map. - CovUpdateMapping(GET_CALLER_PC()); + CovUpdateMapping(coverage_dir, GET_CALLER_PC()); } coverage_data.Extend(npcs); } @@ -571,11 +696,23 @@ uptr __sanitizer_get_total_unique_coverage() { } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_func_enter(uptr *cache) { - coverage_data.TraceBasicaBlock(cache); +void __sanitizer_cov_trace_func_enter(s32 *id) { + coverage_data.TraceBasicBlock(id); +} +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_cov_trace_basic_block(s32 *id) { + coverage_data.TraceBasicBlock(id); +} +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_reset_coverage() { + coverage_data.ReinitializeGuards(); + internal_bzero_aligned16( + coverage_data.data(), + RoundUpTo(coverage_data.size() * sizeof(coverage_data.data()[0]), 16)); } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_basic_block(uptr *cache) { - coverage_data.TraceBasicaBlock(cache); +uptr __sanitizer_get_coverage_guards(uptr **data) { + *data = coverage_data.data(); + return coverage_data.size(); } } // extern "C" |