diff options
Diffstat (limited to 'lib/hwasan')
27 files changed, 847 insertions, 446 deletions
diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cpp index e2bfea5e4226..6f2246552164 100644 --- a/lib/hwasan/hwasan.cc +++ b/lib/hwasan/hwasan.cpp @@ -1,9 +1,8 @@ -//===-- hwasan.cc ---------------------------------------------------------===// +//===-- hwasan.cpp --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "hwasan.h" #include "hwasan_checks.h" +#include "hwasan_dynamic_shadow.h" #include "hwasan_poisoning.h" #include "hwasan_report.h" #include "hwasan_thread.h" @@ -58,7 +58,7 @@ Flags *flags() { } int hwasan_inited = 0; -int hwasan_shadow_inited = 0; +int hwasan_instrumentation_inited = 0; bool hwasan_init_is_running; int hwasan_report_count = 0; @@ -88,6 +88,8 @@ static void InitializeFlags() { cf.check_printf = false; cf.intercept_tls_get_addr = true; cf.exitcode = 99; + // 8 shadow pages ~512kB, small enough to cover common stack sizes. + cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); // Sigtrap is used in error reporting. cf.handle_sigtrap = kHandleSignalExclusive; @@ -127,13 +129,10 @@ static void InitializeFlags() { ubsan_parser.ParseString(ubsan_default_options); #endif - const char *hwasan_options = GetEnv("HWASAN_OPTIONS"); - parser.ParseString(hwasan_options); + parser.ParseStringFromEnv("HWASAN_OPTIONS"); #if HWASAN_CONTAINS_UBSAN - ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); #endif - VPrintf(1, "HWASAN_OPTIONS: %s\n", - hwasan_options ? hwasan_options : "<empty>"); InitializeCommonFlags(); @@ -142,23 +141,6 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); } -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind) { - Thread *t = GetCurrentThread(); - if (!t) { - // the thread is still being created. - stack->size = 0; - return; - } - if (!StackTrace::WillUseFastUnwind(request_fast_unwind)) { - // Block reports from our interceptors during _Unwind_Backtrace. - SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); - } - stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), - request_fast_unwind); -} - static void HWAsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2) { Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, @@ -188,17 +170,13 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) { #if SANITIZER_ANDROID static char *memory_usage_buffer = nullptr; -#define PR_SET_VMA 0x53564d41 -#define PR_SET_VMA_ANON_NAME 0 - static void InitMemoryUsage() { memory_usage_buffer = (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); CHECK(memory_usage_buffer); memory_usage_buffer[0] = '\0'; - CHECK(internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, - (uptr)memory_usage_buffer, kMemoryUsageBufferSize, - (uptr)memory_usage_buffer) == 0); + DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize, + memory_usage_buffer); } void UpdateMemoryUsage() { @@ -215,56 +193,57 @@ void UpdateMemoryUsage() { void UpdateMemoryUsage() {} #endif -struct FrameDescription { - uptr PC; - const char *Descr; -}; - -struct FrameDescriptionArray { - FrameDescription *beg, *end; -}; +// Prepare to run instrumented code on the main thread. +void InitInstrumentation() { + if (hwasan_instrumentation_inited) return; -static InternalMmapVectorNoCtor<FrameDescriptionArray> AllFrames; + if (!InitShadow()) { + Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); + DumpProcessMap(); + Die(); + } -void InitFrameDescriptors(uptr b, uptr e) { - FrameDescription *beg = reinterpret_cast<FrameDescription *>(b); - FrameDescription *end = reinterpret_cast<FrameDescription *>(e); - if (beg == end) - return; - AllFrames.push_back({beg, end}); - if (Verbosity()) - for (FrameDescription *frame_descr = beg; frame_descr < end; frame_descr++) - Printf("Frame: %p %s\n", frame_descr->PC, frame_descr->Descr); -} + InitThreads(); + hwasanThreadList().CreateCurrentThread(); -const char *GetStackFrameDescr(uptr pc) { - for (uptr i = 0, n = AllFrames.size(); i < n; i++) - for (auto p = AllFrames[i].beg; p < AllFrames[i].end; p++) - if (p->PC == pc) - return p->Descr; - return nullptr; + hwasan_instrumentation_inited = 1; } } // namespace __hwasan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __hwasan; + Thread *t = GetCurrentThread(); + if (!t) { + // the thread is still being created. + size = 0; + return; + } + if (!StackTrace::WillUseFastUnwind(request_fast)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return Unwind(max_depth, pc, bp, context, 0, 0, request_fast); + } + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} + // Interface. using namespace __hwasan; uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. -void __hwasan_shadow_init() { - if (hwasan_shadow_inited) return; - if (!InitShadow()) { - Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); - DumpProcessMap(); - Die(); - } - hwasan_shadow_inited = 1; -} +// This function was used by the old frame descriptor mechanism. We keep it +// around to avoid breaking ABI. +void __hwasan_init_frames(uptr beg, uptr end) {} -void __hwasan_init_frames(uptr beg, uptr end) { - InitFrameDescriptors(beg, end); +void __hwasan_init_static() { + InitShadowGOT(); + InitInstrumentation(); } void __hwasan_init() { @@ -287,10 +266,11 @@ void __hwasan_init() { DisableCoreDumperIfNecessary(); - __hwasan_shadow_init(); + InitInstrumentation(); - InitThreads(); - hwasanThreadList().CreateCurrentThread(); + // Needs to be called here because flags()->random_tags might not have been + // initialized when InitInstrumentation() was called. + GetCurrentThread()->InitRandomState(); MadviseShadow(); @@ -335,14 +315,14 @@ sptr __hwasan_test_shadow(const void *p, uptr sz) { if (sz == 0) return -1; tag_t ptr_tag = GetTagFromPointer((uptr)p); - if (ptr_tag == 0) - return -1; uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); uptr shadow_first = MemToShadow(ptr_raw); uptr shadow_last = MemToShadow(ptr_raw + sz - 1); for (uptr s = shadow_first; s <= shadow_last; ++s) - if (*(tag_t*)s != ptr_tag) - return ShadowToMem(s) - ptr_raw; + if (*(tag_t *)s != ptr_tag) { + sptr offset = ShadowToMem(s) - ptr_raw; + return offset < 0 ? 0 : offset; + } return -1; } @@ -467,6 +447,28 @@ void __hwasan_handle_longjmp(const void *sp_dst) { TagMemory(sp, dst - sp, 0); } +void __hwasan_handle_vfork(const void *sp_dst) { + uptr sp = (uptr)sp_dst; + Thread *t = GetCurrentThread(); + CHECK(t); + uptr top = t->stack_top(); + uptr bottom = t->stack_bottom(); + if (top == 0 || bottom == 0 || sp < bottom || sp >= top) { + Report( + "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: " + "stack top: %zx; current %zx; bottom: %zx \n" + "False positive error reports may follow\n", + top, sp, bottom); + return; + } + TagMemory(bottom, sp - bottom, 0); +} + +extern "C" void *__hwasan_extra_spill_area() { + Thread *t = GetCurrentThread(); + return &t->vfork_spill(); +} + void __hwasan_print_memory_usage() { InternalScopedString s(kMemoryUsageBufferSize); HwasanFormatMemoryUsage(s); diff --git a/lib/hwasan/hwasan.h b/lib/hwasan/hwasan.h index ce9e904c5c69..465e56c3a8cc 100644 --- a/lib/hwasan/hwasan.h +++ b/lib/hwasan/hwasan.h @@ -1,9 +1,8 @@ //===-- hwasan.h ------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,6 +44,11 @@ const uptr kAddressTagMask = 0xFFUL << kAddressTagShift; // for threads and stack histories. This is an ABI constant. const unsigned kShadowBaseAlignment = 32; +const unsigned kRecordAddrBaseTagShift = 3; +const unsigned kRecordFPShift = 48; +const unsigned kRecordFPLShift = 4; +const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift); + static inline tag_t GetTagFromPointer(uptr p) { return p >> kAddressTagShift; } @@ -71,6 +75,7 @@ extern int hwasan_report_count; bool ProtectRange(uptr beg, uptr end); bool InitShadow(); void InitThreads(); +void InitInstrumentation(); void MadviseShadow(); char *GetProcSelfMaps(); void InitializeInterceptors(); @@ -81,6 +86,7 @@ void HwasanAllocatorThreadFinish(); void *hwasan_malloc(uptr size, StackTrace *stack); void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack); +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack); void *hwasan_valloc(uptr size, StackTrace *stack); void *hwasan_pvalloc(uptr size, StackTrace *stack); void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); @@ -92,9 +98,6 @@ void hwasan_free(void *ptr, StackTrace *stack); void InstallTrapHandler(); void InstallAtExitHandler(); -const char *GetStackOriginDescr(u32 id, uptr *pc); -const char *GetStackFrameDescr(uptr pc); - void EnterSymbolizer(); void ExitSymbolizer(); bool IsInSymbolizer(); @@ -104,9 +107,6 @@ struct SymbolizerScope { ~SymbolizerScope() { ExitSymbolizer(); } }; -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind); - // Returns a "chained" origin id, pointing to the given stack trace followed by // the previous origin id. u32 ChainOrigin(u32 id, StackTrace *stack); @@ -115,16 +115,15 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_MALLOC_STACK_TRACE \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ - common_flags()->fast_unwind_on_malloc) + if (hwasan_inited) \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ + nullptr, common_flags()->fast_unwind_on_malloc, \ + common_flags()->malloc_context_size) #define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) + if (hwasan_inited) \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_FATAL_STACK_TRACE_HERE \ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()) diff --git a/lib/hwasan/hwasan_allocator.cc b/lib/hwasan/hwasan_allocator.cpp index 8487ed7e1e55..b4fae5820d0a 100644 --- a/lib/hwasan/hwasan_allocator.cc +++ b/lib/hwasan/hwasan_allocator.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_allocator.cc ------------------------- ---------------------===// +//===-- hwasan_allocator.cpp ------------------------ ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,7 +16,9 @@ #include "sanitizer_common/sanitizer_stackdepot.h" #include "hwasan.h" #include "hwasan_allocator.h" +#include "hwasan_checks.h" #include "hwasan_mapping.h" +#include "hwasan_malloc_bisect.h" #include "hwasan_thread.h" #include "hwasan_report.h" @@ -42,13 +43,8 @@ enum RightAlignMode { kRightAlignAlways }; -// These two variables are initialized from flags()->malloc_align_right -// in HwasanAllocatorInit and are never changed afterwards. -static RightAlignMode right_align_mode = kRightAlignNever; -static bool right_align_8 = false; - // Initialized in HwasanAllocatorInit, an never changed. -static ALIGNED(16) u8 tail_magic[kShadowAlignment]; +static ALIGNED(16) u8 tail_magic[kShadowAlignment - 1]; bool HwasanChunkView::IsAllocated() const { return metadata_ && metadata_->alloc_context_id && metadata_->requested_size; @@ -58,8 +54,6 @@ bool HwasanChunkView::IsAllocated() const { static uptr AlignRight(uptr addr, uptr requested_size) { uptr tail_size = requested_size % kShadowAlignment; if (!tail_size) return addr; - if (right_align_8) - return tail_size > 8 ? addr : addr + 8; return addr + kShadowAlignment - tail_size; } @@ -95,30 +89,7 @@ void HwasanAllocatorInit() { !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); allocator.Init(common_flags()->allocator_release_to_os_interval_ms); - switch (flags()->malloc_align_right) { - case 0: break; - case 1: - right_align_mode = kRightAlignSometimes; - right_align_8 = false; - break; - case 2: - right_align_mode = kRightAlignAlways; - right_align_8 = false; - break; - case 8: - right_align_mode = kRightAlignSometimes; - right_align_8 = true; - break; - case 9: - right_align_mode = kRightAlignAlways; - right_align_8 = true; - break; - default: - Report("ERROR: unsupported value of malloc_align_right flag: %d\n", - flags()->malloc_align_right); - Die(); - } - for (uptr i = 0; i < kShadowAlignment; i++) + for (uptr i = 0; i < sizeof(tail_magic); i++) tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); } @@ -172,23 +143,32 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size); internal_memset(allocated, flags()->malloc_fill_byte, fill_size); } - if (!right_align_mode) + if (size != orig_size) { internal_memcpy(reinterpret_cast<u8 *>(allocated) + orig_size, tail_magic, - size - orig_size); + size - orig_size - 1); + } void *user_ptr = allocated; - if (flags()->tag_in_malloc && - atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - user_ptr = (void *)TagMemoryAligned( - (uptr)user_ptr, size, t ? t->GenerateRandomTag() : kFallbackAllocTag); - - if ((orig_size % kShadowAlignment) && (alignment <= kShadowAlignment) && - right_align_mode) { - uptr as_uptr = reinterpret_cast<uptr>(user_ptr); - if (right_align_mode == kRightAlignAlways || - GetTagFromPointer(as_uptr) & 1) { // use a tag bit as a random bit. - user_ptr = reinterpret_cast<void *>(AlignRight(as_uptr, orig_size)); - meta->right_aligned = 1; + // Tagging can only be skipped when both tag_in_malloc and tag_in_free are + // false. When tag_in_malloc = false and tag_in_free = true malloc needs to + // retag to 0. + if ((flags()->tag_in_malloc || flags()->tag_in_free) && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { + if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) { + tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag; + uptr tag_size = orig_size ? orig_size : 1; + uptr full_granule_size = RoundDownTo(tag_size, kShadowAlignment); + user_ptr = + (void *)TagMemoryAligned((uptr)user_ptr, full_granule_size, tag); + if (full_granule_size != tag_size) { + u8 *short_granule = + reinterpret_cast<u8 *>(allocated) + full_granule_size; + TagMemoryAligned((uptr)short_granule, kShadowAlignment, + tag_size % kShadowAlignment); + short_granule[kShadowAlignment - 1] = tag; + } + } else { + user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0); } } @@ -198,10 +178,10 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, static bool PointerAndMemoryTagsMatch(void *tagged_ptr) { CHECK(tagged_ptr); - tag_t ptr_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr)); + uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr); tag_t mem_tag = *reinterpret_cast<tag_t *>( MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr)))); - return ptr_tag == mem_tag; + return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1); } static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { @@ -222,14 +202,15 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { // Check tail magic. uptr tagged_size = TaggedSize(orig_size); - if (flags()->free_checks_tail_magic && !right_align_mode && orig_size) { - uptr tail_size = tagged_size - orig_size; + if (flags()->free_checks_tail_magic && orig_size && + tagged_size != orig_size) { + uptr tail_size = tagged_size - orig_size - 1; CHECK_LT(tail_size, kShadowAlignment); void *tail_beg = reinterpret_cast<void *>( reinterpret_cast<uptr>(aligned_ptr) + orig_size); if (tail_size && internal_memcmp(tail_beg, tail_magic, tail_size)) ReportTailOverwritten(stack, reinterpret_cast<uptr>(tagged_ptr), - orig_size, tail_size, tail_magic); + orig_size, tail_magic); } meta->requested_size = 0; @@ -242,7 +223,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size); internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); } - if (flags()->tag_in_free && + if (flags()->tag_in_free && malloc_bisect(stack, 0) && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size), t ? t->GenerateRandomTag() : kFallbackFreeTag); @@ -335,6 +316,16 @@ void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); } +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return hwasan_realloc(ptr, nmemb * size, stack); +} + void *hwasan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull( HwasanAllocate(stack, size, GetPageSizeCached(), false)); diff --git a/lib/hwasan/hwasan_allocator.h b/lib/hwasan/hwasan_allocator.h index 6ab722fa6bef..3a50a11f3526 100644 --- a/lib/hwasan/hwasan_allocator.h +++ b/lib/hwasan/hwasan_allocator.h @@ -1,9 +1,8 @@ //===-- hwasan_allocator.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,10 +61,8 @@ struct AP64 { static const uptr kFlags = 0; }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; void AllocatorSwallowThreadLocalCache(AllocatorCache *cache); diff --git a/lib/hwasan/hwasan_checks.h b/lib/hwasan/hwasan_checks.h index 688b5e2bed82..a8de0fef20f0 100644 --- a/lib/hwasan/hwasan_checks.h +++ b/lib/hwasan/hwasan_checks.h @@ -1,9 +1,8 @@ //===-- hwasan_checks.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define HWASAN_CHECKS_H #include "hwasan_mapping.h" +#include "sanitizer_common/sanitizer_common.h" namespace __hwasan { template <unsigned X> @@ -23,8 +23,8 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { (void)p; // 0x900 is added to do not interfere with the kernel use of lower values of // brk immediate. - // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. - asm("brk %0\n\t" ::"n"(0x900 + X)); + register uptr x0 asm("x0") = p; + asm("brk %1\n\t" ::"r"(x0), "n"(0x900 + X)); #elif defined(__x86_64__) // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes // total. The pointer is passed via rdi. @@ -42,15 +42,48 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { // __builtin_unreachable(); } +// Version with access size which is not power of 2 +template <unsigned X> +__attribute__((always_inline)) static void SigTrap(uptr p, uptr size) { +#if defined(__aarch64__) + register uptr x0 asm("x0") = p; + register uptr x1 asm("x1") = size; + asm("brk %2\n\t" ::"r"(x0), "r"(x1), "n"(0x900 + X)); +#elif defined(__x86_64__) + // Size is stored in rsi. + asm volatile( + "int3\n" + "nopl %c0(%%rax)\n" ::"n"(0x40 + X), + "D"(p), "S"(size)); +#else + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + +__attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches( + tag_t mem_tag, uptr ptr, uptr sz) { + tag_t ptr_tag = GetTagFromPointer(ptr); + if (ptr_tag == mem_tag) + return true; + if (mem_tag >= kShadowAlignment) + return false; + if ((ptr & (kShadowAlignment - 1)) + sz > mem_tag) + return false; +#ifndef __aarch64__ + ptr = UntagAddr(ptr); +#endif + return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag; +} + enum class ErrorAction { Abort, Recover }; enum class AccessType { Load, Store }; template <ErrorAction EA, AccessType AT, unsigned LogSize> __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { - tag_t ptr_tag = GetTagFromPointer(p); uptr ptr_raw = p & ~kAddressTagMask; tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw); - if (UNLIKELY(ptr_tag != mem_tag)) { + if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) { SigTrap<0x20 * (EA == ErrorAction::Recover) + 0x10 * (AT == AccessType::Store) + LogSize>(p); if (EA == ErrorAction::Abort) @@ -66,15 +99,26 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, tag_t ptr_tag = GetTagFromPointer(p); uptr ptr_raw = p & ~kAddressTagMask; tag_t *shadow_first = (tag_t *)MemToShadow(ptr_raw); - tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz - 1); - for (tag_t *t = shadow_first; t <= shadow_last; ++t) + tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz); + for (tag_t *t = shadow_first; t < shadow_last; ++t) if (UNLIKELY(ptr_tag != *t)) { SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + 0xf>(p); + 0x10 * (AT == AccessType::Store) + 0xf>(p, sz); if (EA == ErrorAction::Abort) __builtin_unreachable(); } + uptr end = p + sz; + uptr tail_sz = end & 0xf; + if (UNLIKELY(tail_sz != 0 && + !PossiblyShortTagMatches( + *shadow_last, end & ~(kShadowAlignment - 1), tail_sz))) { + SigTrap<0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + 0xf>(p, sz); + if (EA == ErrorAction::Abort) + __builtin_unreachable(); + } } + } // end namespace __hwasan #endif // HWASAN_CHECKS_H diff --git a/lib/hwasan/hwasan_dynamic_shadow.cc b/lib/hwasan/hwasan_dynamic_shadow.cpp index 87670f508283..a04751f44a31 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.cc +++ b/lib/hwasan/hwasan_dynamic_shadow.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_dynamic_shadow.cc --------------------------------*- C++ -*-===// +//===-- hwasan_dynamic_shadow.cpp -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -19,6 +18,9 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_posix.h" +#include <elf.h> +#include <link.h> + // The code in this file needs to run in an unrelocated binary. It should not // access any external symbol, including its own non-hidden globals. @@ -119,10 +121,28 @@ decltype(__hwasan_shadow)* __hwasan_premap_shadow() { INTERFACE_ATTRIBUTE __attribute__((ifunc("__hwasan_premap_shadow"))) void __hwasan_shadow(); +extern __attribute((weak, visibility("hidden"))) ElfW(Rela) __rela_iplt_start[], + __rela_iplt_end[]; + } // extern "C" namespace __hwasan { +void InitShadowGOT() { + // Call the ifunc resolver for __hwasan_shadow and fill in its GOT entry. This + // needs to be done before other ifunc resolvers (which are handled by libc) + // because a resolver might read __hwasan_shadow. + typedef ElfW(Addr) (*ifunc_resolver_t)(void); + for (ElfW(Rela) *r = __rela_iplt_start; r != __rela_iplt_end; ++r) { + ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset); + ElfW(Addr) resolver = r->r_addend; + if (resolver == reinterpret_cast<ElfW(Addr)>(&__hwasan_premap_shadow)) { + *offset = reinterpret_cast<ifunc_resolver_t>(resolver)(); + break; + } + } +} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { if (IsPremapShadowAvailable()) return FindPremappedShadowStart(shadow_size_bytes); @@ -133,10 +153,12 @@ uptr FindDynamicShadowStart(uptr shadow_size_bytes) { #else namespace __hwasan { +void InitShadowGOT() {} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { return MapDynamicShadow(shadow_size_bytes); } } // namespace __hwasan -# + #endif // SANITIZER_ANDROID diff --git a/lib/hwasan/hwasan_dynamic_shadow.h b/lib/hwasan/hwasan_dynamic_shadow.h index b5e9e1dd6a06..3c2e7c716a34 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.h +++ b/lib/hwasan/hwasan_dynamic_shadow.h @@ -1,9 +1,8 @@ //===-- hwasan_dynamic_shadow.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -21,6 +20,7 @@ namespace __hwasan { uptr FindDynamicShadowStart(uptr shadow_size_bytes); +void InitShadowGOT(); } // namespace __hwasan diff --git a/lib/hwasan/hwasan_flags.h b/lib/hwasan/hwasan_flags.h index 492d5bb98c27..0a6998f675d6 100644 --- a/lib/hwasan/hwasan_flags.h +++ b/lib/hwasan/hwasan_flags.h @@ -1,9 +1,8 @@ //===-- hwasan_flags.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_flags.inc b/lib/hwasan/hwasan_flags.inc index b450ab9503f9..2dff2b9aca6e 100644 --- a/lib/hwasan/hwasan_flags.inc +++ b/lib/hwasan/hwasan_flags.inc @@ -1,9 +1,8 @@ //===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,32 +37,6 @@ HWASAN_FLAG( "HWASan allocator flag. max_malloc_fill_size is the maximal amount of " "bytes that will be filled with malloc_fill_byte on malloc.") -// Rules for malloc alignment on aarch64: -// * If the size is 16-aligned, then malloc should return 16-aligned memory. -// * Otherwise, malloc should return 8-alignment memory. -// So, -// * If the size is 16-aligned, we don't need to do anything. -// * Otherwise we don't have to obey 16-alignment, just the 8-alignment. -// * We may want to break the 8-alignment rule to catch more buffer overflows -// but this will break valid code in some rare cases, like this: -// struct Foo { -// // accessed via atomic instructions that require 8-alignment. -// std::atomic<int64_t> atomic_stuff; -// ... -// char vla[1]; // the actual size of vla could be anything. -// } -// Which means that the safe values for malloc_align_right are 0, 8, 9, -// and the values 1 and 2 may require changes in otherwise valid code. - -HWASAN_FLAG( - int, malloc_align_right, 0, // off by default - "HWASan allocator flag. " - "0 (default): allocations are always aligned left to 16-byte boundary; " - "1: allocations are sometimes aligned right to 1-byte boundary (risky); " - "2: allocations are always aligned right to 1-byte boundary (risky); " - "8: allocations are sometimes aligned right to 8-byte boundary; " - "9: allocations are always aligned right to 8-byte boundary." - ) HWASAN_FLAG(bool, free_checks_tail_magic, 1, "If set, free() will check the magic values " "to the right of the allocated object " @@ -86,3 +59,16 @@ HWASAN_FLAG(int, stack_history_size, 1024, "The number of stack frames remembered per thread. " "Affects the quality of stack-related reports, but not the ability " "to find bugs.") + +// Malloc / free bisection. Only tag malloc and free calls when a hash of +// allocation size and stack trace is between malloc_bisect_left and +// malloc_bisect_right (both inclusive). [0, 0] range is special and disables +// bisection (i.e. everything is tagged). Once the range is narrowed down +// enough, use malloc_bisect_dump to see interesting allocations. +HWASAN_FLAG(uptr, malloc_bisect_left, 0, + "Left bound of malloc bisection, inclusive.") +HWASAN_FLAG(uptr, malloc_bisect_right, 0, + "Right bound of malloc bisection, inclusive.") +HWASAN_FLAG(bool, malloc_bisect_dump, false, + "Print all allocations within [malloc_bisect_left, " + "malloc_bisect_right] range ") diff --git a/lib/hwasan/hwasan_interceptors.cc b/lib/hwasan/hwasan_interceptors.cpp index fb0dcb8905c0..47fed0fc9abb 100644 --- a/lib/hwasan/hwasan_interceptors.cc +++ b/lib/hwasan/hwasan_interceptors.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_interceptors.cc --------------------------------------------===// +//===-- hwasan_interceptors.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,24 +44,6 @@ using __sanitizer::atomic_load; using __sanitizer::atomic_store; using __sanitizer::atomic_uintptr_t; -bool IsInInterceptorScope() { - Thread *t = GetCurrentThread(); - return t && t->InInterceptorScope(); -} - -struct InterceptorScope { - InterceptorScope() { - Thread *t = GetCurrentThread(); - if (t) - t->EnterInterceptorScope(); - } - ~InterceptorScope() { - Thread *t = GetCurrentThread(); - if (t) - t->LeaveInterceptorScope(); - } -}; - static uptr allocated_for_dlsym; static const uptr kDlsymAllocPoolSize = 1024; static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; @@ -179,6 +160,11 @@ void * __sanitizer_realloc(void *ptr, uptr size) { return hwasan_realloc(ptr, size, &stack); } +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_reallocarray(ptr, nmemb, size, &stack); +} + void * __sanitizer_malloc(uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_init_is_running)) @@ -205,6 +191,7 @@ INTERCEPTOR_ALIAS(void, free, void *ptr); INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD @@ -228,6 +215,11 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, } #endif +#if HWASAN_WITH_INTERCEPTORS +DEFINE_REAL(int, vfork) +DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork) +#endif + static void BeforeFork() { StackDepotLockAll(); } @@ -244,11 +236,6 @@ INTERCEPTOR(int, fork, void) { return pid; } - -struct HwasanInterceptorContext { - bool in_interceptor_scope; -}; - namespace __hwasan { int OnExit() { @@ -267,9 +254,12 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(fork); #if HWASAN_WITH_INTERCEPTORS +#if defined(__linux__) + INTERCEPT_FUNCTION(vfork); +#endif // __linux__ #if !defined(__aarch64__) INTERCEPT_FUNCTION(pthread_create); -#endif +#endif // __aarch64__ INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); #endif diff --git a/lib/hwasan/hwasan_interceptors_vfork.S b/lib/hwasan/hwasan_interceptors_vfork.S new file mode 100644 index 000000000000..13d0829c0865 --- /dev/null +++ b/lib/hwasan/hwasan_interceptors_vfork.S @@ -0,0 +1,10 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__linux__) && HWASAN_WITH_INTERCEPTORS +#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area +#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork +#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" +#endif + +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_interface_internal.h b/lib/hwasan/hwasan_interface_internal.h index d3b2087d03c6..1b10d76c78e9 100644 --- a/lib/hwasan/hwasan_interface_internal.h +++ b/lib/hwasan/hwasan_interface_internal.h @@ -1,9 +1,8 @@ //===-- hwasan_interface_internal.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,7 +20,7 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE -void __hwasan_shadow_init(); +void __hwasan_init_static(); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_init(); @@ -101,6 +100,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __hwasan_tag_pointer(uptr p, u8 tag); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_tag_mismatch(uptr addr, u8 ts); + +SANITIZER_INTERFACE_ATTRIBUTE u8 __hwasan_generate_tag(); // Returns the offset of the first tag mismatch or -1 if the whole range is @@ -118,6 +120,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_handle_longjmp(const void *sp_dst); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_handle_vfork(const void *sp_dst); + +SANITIZER_INTERFACE_ATTRIBUTE u16 __sanitizer_unaligned_load16(const uu16 *p); SANITIZER_INTERFACE_ATTRIBUTE @@ -193,6 +198,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_realloc(void *ptr, uptr size); SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_malloc(uptr size); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/lib/hwasan/hwasan_linux.cc b/lib/hwasan/hwasan_linux.cpp index 5b0a8b4ac98f..d932976489e9 100644 --- a/lib/hwasan/hwasan_linux.cc +++ b/lib/hwasan/hwasan_linux.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_linux.cc -----------------------------------------*- C++ -*-===// +//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -39,7 +38,17 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_procmaps.h" -#if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID +// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID. +// +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF +// Not currently tested. +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON +// Integration tests downstream exist. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF +// Tested with check-hwasan on x86_64-linux. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON +// Tested with check-hwasan on aarch64-linux-android. +#if !SANITIZER_ANDROID SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL uptr __hwasan_tls; #endif @@ -219,6 +228,8 @@ bool MemIsApp(uptr p) { } static void HwasanAtExit(void) { + if (common_flags()->print_module_map) + DumpProcessMap(); if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) ReportStats(); if (hwasan_report_count > 0) { @@ -235,7 +246,7 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 extern "C" void __hwasan_thread_enter() { - hwasanThreadList().CreateCurrentThread(); + hwasanThreadList().CreateCurrentThread()->InitRandomState(); } extern "C" void __hwasan_thread_exit() { @@ -288,7 +299,9 @@ uptr *GetCurrentThreadLongPtr() { #if SANITIZER_ANDROID void AndroidTestTlsSlot() { uptr kMagicValue = 0x010203040A0B0C0D; - *(uptr *)get_android_tls_ptr() = kMagicValue; + uptr *tls_ptr = GetCurrentThreadLongPtr(); + uptr old_value = *tls_ptr; + *tls_ptr = kMagicValue; dlerror(); if (*(uptr *)get_android_tls_ptr() != kMagicValue) { Printf( @@ -296,6 +309,7 @@ void AndroidTestTlsSlot() { "for dlerror().\n"); Die(); } + *tls_ptr = old_value; } #else void AndroidTestTlsSlot() {} @@ -369,22 +383,35 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { return AccessInfo{addr, size, is_store, !is_store, recover}; } -static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { - AccessInfo ai = GetAccessInfo(info, uc); - if (!ai.is_store && !ai.is_load) - return false; - +static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, + ucontext_t *uc, uptr *registers_frame = nullptr) { InternalMmapVector<BufferedStackTrace> stack_buffer(1); BufferedStackTrace *stack = stack_buffer.data(); stack->Reset(); - SignalContext sig{info, uc}; - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, uc, common_flags()->fast_unwind_on_fatal); + stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); - ++hwasan_report_count; + // The second stack frame contains the failure __hwasan_check function, as + // we have a stack frame for the registers saved in __hwasan_tag_mismatch that + // we wish to ignore. This (currently) only occurs on AArch64, as x64 + // implementations use SIGTRAP to implement the failure, and thus do not go + // through the stack saver. + if (registers_frame && stack->trace && stack->size > 0) { + stack->trace++; + stack->size--; + } bool fatal = flags()->halt_on_error || !ai.recover; - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal); + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, + registers_frame); +} + +static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { + AccessInfo ai = GetAccessInfo(info, uc); + if (!ai.is_store && !ai.is_load) + return false; + + SignalContext sig{info, uc}; + HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc); #if defined(__aarch64__) uc->uc_mcontext.pc += 4; @@ -395,10 +422,25 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { return true; } +// Entry point stub for interoperability between __hwasan_tag_mismatch (ASM) and +// the rest of the mismatch handling code (C++). +extern "C" void __hwasan_tag_mismatch_stub(uptr addr, uptr access_info, + uptr *registers_frame) { + AccessInfo ai; + ai.is_store = access_info & 0x10; + ai.recover = false; + ai.addr = addr; + ai.size = 1 << (access_info & 0xf); + + HandleTagMismatch(ai, (uptr)__builtin_return_address(0), + (uptr)__builtin_frame_address(0), nullptr, registers_frame); + __builtin_unreachable(); +} + static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); + stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } void HwasanOnDeadlySignal(int signo, void *info, void *context) { diff --git a/lib/hwasan/hwasan_malloc_bisect.h b/lib/hwasan/hwasan_malloc_bisect.h new file mode 100644 index 000000000000..eaf124aab7dd --- /dev/null +++ b/lib/hwasan/hwasan_malloc_bisect.h @@ -0,0 +1,50 @@ +//===-- hwasan_malloc_bisect.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_hash.h" +#include "hwasan.h" + +namespace __hwasan { + +static u32 malloc_hash(StackTrace *stack, uptr orig_size) { + uptr len = Min(stack->size, (unsigned)7); + MurMur2HashBuilder H(len); + H.add(orig_size); + // Start with frame #1 to skip __sanitizer_malloc frame, which is + // (a) almost always the same (well, could be operator new or new[]) + // (b) can change hashes when compiler-rt is rebuilt, invalidating previous + // bisection results. + // Because of ASLR, use only offset inside the page. + for (uptr i = 1; i < len; ++i) H.add(((u32)stack->trace[i]) & 0xFFF); + return H.get(); +} + +static INLINE bool malloc_bisect(StackTrace *stack, uptr orig_size) { + uptr left = flags()->malloc_bisect_left; + uptr right = flags()->malloc_bisect_right; + if (LIKELY(left == 0 && right == 0)) + return true; + if (!stack) + return true; + // Allow malloc_bisect_right > (u32)(-1) to avoid spelling the latter in + // decimal. + uptr h = (uptr)malloc_hash(stack, orig_size); + if (h < left || h > right) + return false; + if (flags()->malloc_bisect_dump) { + Printf("[alloc] %u %zu\n", h, orig_size); + stack->Print(); + } + return true; +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_mapping.h b/lib/hwasan/hwasan_mapping.h index e5e23dc60337..a86ad7ca8036 100644 --- a/lib/hwasan/hwasan_mapping.h +++ b/lib/hwasan/hwasan_mapping.h @@ -1,9 +1,8 @@ //===-- hwasan_mapping.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_memintrinsics.cc b/lib/hwasan/hwasan_memintrinsics.cpp index 9cb844e45c40..e82d77a1bc16 100644 --- a/lib/hwasan/hwasan_memintrinsics.cc +++ b/lib/hwasan/hwasan_memintrinsics.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_memintrinsics.cc ---------------------------------*- C++ -*-===// +//===-- hwasan_memintrinsics.cpp --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_new_delete.cc b/lib/hwasan/hwasan_new_delete.cpp index f2e8faf5da73..4a9c79fe41b3 100644 --- a/lib/hwasan/hwasan_new_delete.cc +++ b/lib/hwasan/hwasan_new_delete.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_new_delete.cc ----------------------------------------------===// +//===-- hwasan_new_delete.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,15 +35,15 @@ namespace std { if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\ return res -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } @@ -53,13 +52,13 @@ void *operator new[](size_t size, std::nothrow_t const&) { GET_MALLOC_STACK_TRACE; \ if (ptr) hwasan_free(ptr, &stack) -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc deleted file mode 100644 index 9c8e16b12ad5..000000000000 --- a/lib/hwasan/hwasan_poisoning.cc +++ /dev/null @@ -1,37 +0,0 @@ -//===-- hwasan_poisoning.cc -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of HWAddressSanitizer. -// -//===----------------------------------------------------------------------===// - -#include "hwasan_poisoning.h" - -#include "hwasan_mapping.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" - -namespace __hwasan { - -uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { - CHECK(IsAligned(p, kShadowAlignment)); - CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MemToShadow(p); - uptr shadow_size = MemToShadowSize(size); - internal_memset((void *)shadow_start, tag, shadow_size); - return AddTagToPointer(p, tag); -} - -uptr TagMemory(uptr p, uptr size, tag_t tag) { - uptr start = RoundDownTo(p, kShadowAlignment); - uptr end = RoundUpTo(p + size, kShadowAlignment); - return TagMemoryAligned(start, end - start, tag); -} - -} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.cpp b/lib/hwasan/hwasan_poisoning.cpp new file mode 100644 index 000000000000..2a0816428e75 --- /dev/null +++ b/lib/hwasan/hwasan_poisoning.cpp @@ -0,0 +1,52 @@ +//===-- hwasan_poisoning.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "hwasan_poisoning.h" + +#include "hwasan_mapping.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_linux.h" + +namespace __hwasan { + +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); + + uptr page_size = GetPageSizeCached(); + uptr page_start = RoundUpTo(shadow_start, page_size); + uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); + uptr threshold = common_flags()->clear_shadow_mmap_threshold; + if (SANITIZER_LINUX && + UNLIKELY(page_end >= page_start + threshold && tag == 0)) { + internal_memset((void *)shadow_start, tag, page_start - shadow_start); + internal_memset((void *)page_end, tag, + shadow_start + shadow_size - page_end); + // For an anonymous private mapping MADV_DONTNEED will return a zero page on + // Linux. + ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); + } else { + internal_memset((void *)shadow_start, tag, shadow_size); + } + return AddTagToPointer(p, tag); +} + +uptr TagMemory(uptr p, uptr size, tag_t tag) { + uptr start = RoundDownTo(p, kShadowAlignment); + uptr end = RoundUpTo(p + size, kShadowAlignment); + return TagMemoryAligned(start, end - start, tag); +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.h b/lib/hwasan/hwasan_poisoning.h index 0dbf9d8ed4ef..61751f7d252e 100644 --- a/lib/hwasan/hwasan_poisoning.h +++ b/lib/hwasan/hwasan_poisoning.h @@ -1,9 +1,8 @@ //===-- hwasan_poisoning.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_report.cc b/lib/hwasan/hwasan_report.cpp index ea3e4096daef..346889797888 100644 --- a/lib/hwasan/hwasan_report.cc +++ b/lib/hwasan/hwasan_report.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_report.cc --------------------------------------------------===// +//===-- hwasan_report.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_report.h" #include "hwasan_thread.h" #include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" @@ -35,15 +35,21 @@ class ScopedReport { ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) { BlockingMutexLock lock(&error_message_lock_); error_message_ptr_ = fatal ? &error_message_ : nullptr; + ++hwasan_report_count; } ~ScopedReport() { - BlockingMutexLock lock(&error_message_lock_); - if (fatal) { - SetAbortMessage(error_message_.data()); - Die(); + { + BlockingMutexLock lock(&error_message_lock_); + if (fatal) + SetAbortMessage(error_message_.data()); + error_message_ptr_ = nullptr; } - error_message_ptr_ = nullptr; + if (common_flags()->print_module_map >= 2 || + (fatal && common_flags()->print_module_map)) + DumpProcessMap(); + if (fatal) + Die(); } static void MaybeAppendToErrorMessage(const char *msg) { @@ -133,6 +139,88 @@ uptr FindHeapAllocation(HeapAllocationsRingBuffer *rb, return 0; } +static void PrintStackAllocations(StackAllocationsRingBuffer *sa, + tag_t addr_tag, uptr untagged_addr) { + uptr frames = Min((uptr)flags()->stack_history_size, sa->size()); + bool found_local = false; + for (uptr i = 0; i < frames; i++) { + const uptr *record_addr = &(*sa)[i]; + uptr record = *record_addr; + if (!record) + break; + tag_t base_tag = + reinterpret_cast<uptr>(record_addr) >> kRecordAddrBaseTagShift; + uptr fp = (record >> kRecordFPShift) << kRecordFPLShift; + uptr pc_mask = (1ULL << kRecordFPShift) - 1; + uptr pc = record & pc_mask; + FrameInfo frame; + if (Symbolizer::GetOrInit()->SymbolizeFrame(pc, &frame)) { + for (LocalInfo &local : frame.locals) { + if (!local.has_frame_offset || !local.has_size || !local.has_tag_offset) + continue; + tag_t obj_tag = base_tag ^ local.tag_offset; + if (obj_tag != addr_tag) + continue; + // Calculate the offset from the object address to the faulting + // address. Because we only store bits 4-19 of FP (bits 0-3 are + // guaranteed to be zero), the calculation is performed mod 2^20 and may + // harmlessly underflow if the address mod 2^20 is below the object + // address. + uptr obj_offset = + (untagged_addr - fp - local.frame_offset) & (kRecordFPModulus - 1); + if (obj_offset >= local.size) + continue; + if (!found_local) { + Printf("Potentially referenced stack objects:\n"); + found_local = true; + } + Printf(" %s in %s %s:%d\n", local.name, local.function_name, + local.decl_file, local.decl_line); + } + frame.Clear(); + } + } + + if (found_local) + return; + + // We didn't find any locals. Most likely we don't have symbols, so dump + // the information that we have for offline analysis. + InternalScopedString frame_desc(GetPageSizeCached() * 2); + Printf("Previously allocated frames:\n"); + for (uptr i = 0; i < frames; i++) { + const uptr *record_addr = &(*sa)[i]; + uptr record = *record_addr; + if (!record) + break; + uptr pc_mask = (1ULL << 48) - 1; + uptr pc = record & pc_mask; + frame_desc.append(" record_addr:0x%zx record:0x%zx", + reinterpret_cast<uptr>(record_addr), record); + if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { + RenderFrame(&frame_desc, " %F %L\n", 0, frame->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + frame->ClearAll(); + } + Printf("%s", frame_desc.data()); + frame_desc.clear(); + } +} + +// Returns true if tag == *tag_ptr, reading tags from short granules if +// necessary. This may return a false positive if tags 1-15 are used as a +// regular tag rather than a short granule marker. +static bool TagsEqual(tag_t tag, tag_t *tag_ptr) { + if (tag == *tag_ptr) + return true; + if (*tag_ptr == 0 || *tag_ptr > kShadowAlignment - 1) + return false; + uptr mem = ShadowToMem(reinterpret_cast<uptr>(tag_ptr)); + tag_t inline_tag = *reinterpret_cast<tag_t *>(mem + kShadowAlignment - 1); + return tag == inline_tag; +} + void PrintAddressDescription( uptr tagged_addr, uptr access_size, StackAllocationsRingBuffer *current_stack_allocations) { @@ -160,39 +248,36 @@ void PrintAddressDescription( // check the allocator if it has a live chunk there. tag_t addr_tag = GetTagFromPointer(tagged_addr); tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); - if (*tag_ptr != addr_tag) { // should be true usually. - tag_t *left = tag_ptr, *right = tag_ptr; - // scan left. - for (int i = 0; i < 1000 && *left == *tag_ptr; i++, left--){} - // scan right. - for (int i = 0; i < 1000 && *right == *tag_ptr; i++, right++){} - // Chose the object that has addr_tag and that is closer to addr. - tag_t *candidate = nullptr; - if (*right == addr_tag && *left == addr_tag) - candidate = right - tag_ptr < tag_ptr - left ? right : left; - else if (*right == addr_tag) - candidate = right; - else if (*left == addr_tag) + tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr; + for (int i = 0; i < 1000; i++) { + if (TagsEqual(addr_tag, left)) { candidate = left; + break; + } + --left; + if (TagsEqual(addr_tag, right)) { + candidate = right; + break; + } + ++right; + } - if (candidate) { - uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); - HwasanChunkView chunk = FindHeapChunkByAddress(mem); - if (chunk.IsAllocated()) { - Printf("%s", d.Location()); - Printf( - "%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", - untagged_addr, - candidate == left ? untagged_addr - chunk.End() - : chunk.Beg() - untagged_addr, - candidate == right ? "left" : "right", chunk.UsedSize(), - chunk.Beg(), chunk.End()); - Printf("%s", d.Allocation()); - Printf("allocated here:\n"); - Printf("%s", d.Default()); - GetStackTraceFromId(chunk.GetAllocStackId()).Print(); - num_descriptions_printed++; - } + if (candidate) { + uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); + HwasanChunkView chunk = FindHeapChunkByAddress(mem); + if (chunk.IsAllocated()) { + Printf("%s", d.Location()); + Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", + untagged_addr, + candidate == left ? untagged_addr - chunk.End() + : chunk.Beg() - untagged_addr, + candidate == left ? "right" : "left", chunk.UsedSize(), + chunk.Beg(), chunk.End()); + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + num_descriptions_printed++; } } @@ -232,33 +317,10 @@ void PrintAddressDescription( Printf("%s", d.Default()); t->Announce(); - // Temporary report section, needs to be improved. - Printf("Previously allocated frames:\n"); auto *sa = (t == GetCurrentThread() && current_stack_allocations) ? current_stack_allocations : t->stack_allocations(); - uptr frames = Min((uptr)flags()->stack_history_size, sa->size()); - InternalScopedString frame_desc(GetPageSizeCached() * 2); - for (uptr i = 0; i < frames; i++) { - uptr record = (*sa)[i]; - if (!record) - break; - uptr sp = (record >> 48) << 4; - uptr pc_mask = (1ULL << 48) - 1; - uptr pc = record & pc_mask; - if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { - frame_desc.append(" sp: 0x%zx pc: %p ", sp, pc); - RenderFrame(&frame_desc, "in %f %s:%l\n", 0, frame->info, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); - frame->ClearAll(); - if (auto Descr = GetStackFrameDescr(pc)) - frame_desc.append(" %s\n", Descr); - } - Printf("%s", frame_desc.data()); - frame_desc.clear(); - } - + PrintStackAllocations(sa, addr_tag, untagged_addr); num_descriptions_printed++; } }); @@ -273,23 +335,20 @@ void PrintAddressDescription( void ReportStats() {} -static void PrintTagsAroundAddr(tag_t *tag_ptr) { - Printf( - "Memory tags around the buggy address (one tag corresponds to %zd " - "bytes):\n", kShadowAlignment); - +static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows, + void (*print_tag)(InternalScopedString &s, + tag_t *tag)) { const uptr row_len = 16; // better be power of two. - const uptr num_rows = 17; tag_t *center_row_beg = reinterpret_cast<tag_t *>( RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len)); tag_t *beg_row = center_row_beg - row_len * (num_rows / 2); - tag_t *end_row = center_row_beg + row_len * (num_rows / 2); + tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2); InternalScopedString s(GetPageSizeCached() * 8); for (tag_t *row = beg_row; row < end_row; row += row_len) { s.append("%s", row == center_row_beg ? "=>" : " "); for (uptr i = 0; i < row_len; i++) { s.append("%s", row + i == tag_ptr ? "[" : " "); - s.append("%02x", row[i]); + print_tag(s, &row[i]); s.append("%s", row + i == tag_ptr ? "]" : " "); } s.append("%s\n", row == center_row_beg ? "<=" : " "); @@ -297,6 +356,34 @@ static void PrintTagsAroundAddr(tag_t *tag_ptr) { Printf("%s", s.data()); } +static void PrintTagsAroundAddr(tag_t *tag_ptr) { + Printf( + "Memory tags around the buggy address (one tag corresponds to %zd " + "bytes):\n", kShadowAlignment); + PrintTagInfoAroundAddr(tag_ptr, 17, [](InternalScopedString &s, tag_t *tag) { + s.append("%02x", *tag); + }); + + Printf( + "Tags for short granules around the buggy address (one tag corresponds " + "to %zd bytes):\n", + kShadowAlignment); + PrintTagInfoAroundAddr(tag_ptr, 3, [](InternalScopedString &s, tag_t *tag) { + if (*tag >= 1 && *tag <= kShadowAlignment) { + uptr granule_addr = ShadowToMem(reinterpret_cast<uptr>(tag)); + s.append("%02x", + *reinterpret_cast<u8 *>(granule_addr + kShadowAlignment - 1)); + } else { + s.append(".."); + } + }); + Printf( + "See " + "https://clang.llvm.org/docs/" + "HardwareAssistedAddressSanitizerDesign.html#short-granules for a " + "description of short granule tags\n"); +} + void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { ScopedReport R(flags()->halt_on_error); @@ -324,7 +411,8 @@ void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { } void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, - uptr tail_size, const u8 *expected) { + const u8 *expected) { + uptr tail_size = kShadowAlignment - (orig_size % kShadowAlignment); ScopedReport R(flags()->halt_on_error); Decorator d; uptr untagged_addr = UntagAddr(tagged_addr); @@ -368,11 +456,9 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, "to the right of a heap object, but within the %zd-byte granule, e.g.\n" " char *x = new char[20];\n" " x[25] = 42;\n" - "By default %s does not detect such bugs at the time of write,\n" - "but can detect them at the time of free/delete.\n" - "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0;\n" - "To enable checking at the time of access, set " - "HWASAN_OPTIONS=malloc_align_right to non-zero\n\n", + "%s does not detect such bugs in uninstrumented code at the time of write," + "\nbut can detect them at the time of free/delete.\n" + "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0\n", kShadowAlignment, SanitizerToolName); Printf("%s", s.data()); GetCurrentThread()->Announce(); @@ -384,7 +470,7 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, } void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, - bool is_store, bool fatal) { + bool is_store, bool fatal, uptr *registers_frame) { ScopedReport R(fatal); SavedStackAllocations current_stack_allocations( GetCurrentThread()->stack_allocations()); @@ -400,13 +486,21 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, Thread *t = GetCurrentThread(); + sptr offset = + __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size); + CHECK(offset >= 0 && offset < static_cast<sptr>(access_size)); tag_t ptr_tag = GetTagFromPointer(tagged_addr); - tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + tag_t *tag_ptr = + reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset)); tag_t mem_tag = *tag_ptr; + Printf("%s", d.Access()); Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n", is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, mem_tag, t->unique_id()); + if (offset != 0) + Printf("Invalid access starting at offset [%zu, %zu)\n", offset, + Min(access_size, static_cast<uptr>(offset) + (1 << kShadowScale))); Printf("%s", d.Default()); stack->Print(); @@ -417,7 +511,37 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, PrintTagsAroundAddr(tag_ptr); + if (registers_frame) + ReportRegisters(registers_frame, pc); + ReportErrorSummary(bug_type, stack); } +// See the frame breakdown defined in __hwasan_tag_mismatch (from +// hwasan_tag_mismatch_aarch64.S). +void ReportRegisters(uptr *frame, uptr pc) { + Printf("Registers where the failure occurred (pc %p):\n", pc); + + // We explicitly print a single line (4 registers/line) each iteration to + // reduce the amount of logcat error messages printed. Each Printf() will + // result in a new logcat line, irrespective of whether a newline is present, + // and so we wish to reduce the number of Printf() calls we have to make. + Printf(" x0 %016llx x1 %016llx x2 %016llx x3 %016llx\n", + frame[0], frame[1], frame[2], frame[3]); + Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n", + frame[4], frame[5], frame[6], frame[7]); + Printf(" x8 %016llx x9 %016llx x10 %016llx x11 %016llx\n", + frame[8], frame[9], frame[10], frame[11]); + Printf(" x12 %016llx x13 %016llx x14 %016llx x15 %016llx\n", + frame[12], frame[13], frame[14], frame[15]); + Printf(" x16 %016llx x17 %016llx x18 %016llx x19 %016llx\n", + frame[16], frame[17], frame[18], frame[19]); + Printf(" x20 %016llx x21 %016llx x22 %016llx x23 %016llx\n", + frame[20], frame[21], frame[22], frame[23]); + Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n", + frame[24], frame[25], frame[26], frame[27]); + Printf(" x28 %016llx x29 %016llx x30 %016llx\n", + frame[28], frame[29], frame[30]); +} + } // namespace __hwasan diff --git a/lib/hwasan/hwasan_report.h b/lib/hwasan/hwasan_report.h index 10fb20cc5be1..de86c38fc01f 100644 --- a/lib/hwasan/hwasan_report.h +++ b/lib/hwasan/hwasan_report.h @@ -1,9 +1,8 @@ //===-- hwasan_report.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -23,11 +22,11 @@ namespace __hwasan { void ReportStats(); void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store, bool fatal); + bool is_store, bool fatal, uptr *registers_frame); void ReportInvalidFree(StackTrace *stack, uptr addr); void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, - uptr tail_size, const u8 *expected); - + const u8 *expected); +void ReportRegisters(uptr *registers_frame, uptr pc); void ReportAtExitStatistics(); diff --git a/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/lib/hwasan/hwasan_tag_mismatch_aarch64.S new file mode 100644 index 000000000000..92f627480486 --- /dev/null +++ b/lib/hwasan/hwasan_tag_mismatch_aarch64.S @@ -0,0 +1,106 @@ +#include "sanitizer_common/sanitizer_asm.h" + +// The content of this file is AArch64-only: +#if defined(__aarch64__) + +// The responsibility of the HWASan entry point in compiler-rt is to primarily +// readjust the stack from the callee and save the current register values to +// the stack. +// This entry point function should be called from a __hwasan_check_* symbol. +// These are generated during a lowering pass in the backend, and are found in +// AArch64AsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for +// further information. +// The __hwasan_check_* caller of this function should have expanded the stack +// and saved the previous values of x0, x1, x29, and x30. This function will +// "consume" these saved values and treats it as part of its own stack frame. +// In this sense, the __hwasan_check_* callee and this function "share" a stack +// frame. This allows us to omit having unwinding information (.cfi_*) present +// in every __hwasan_check_* function, therefore reducing binary size. This is +// particularly important as hwasan_check_* instances are duplicated in every +// translation unit where HWASan is enabled. +// This function calls HwasanTagMismatch to step back into the C++ code that +// completes the stack unwinding and error printing. This function is is not +// permitted to return. + + +// Frame from __hwasan_check_: +// | ... | +// | ... | +// | Previous stack frames... | +// +=================================+ +// | Unused 8-bytes for maintaining | +// | 16-byte SP alignment. | +// +---------------------------------+ +// | Return address (x30) for caller | +// | of __hwasan_check_*. | +// +---------------------------------+ +// | Frame address (x29) for caller | +// | of __hwasan_check_* | +// +---------------------------------+ <-- [SP + 232] +// | ... | +// | | +// | Stack frame space for x2 - x28. | +// | | +// | ... | +// +---------------------------------+ <-- [SP + 16] +// | | +// | Saved x1, as __hwasan_check_* | +// | clobbers it. | +// +---------------------------------+ +// | Saved x0, likewise above. | +// +---------------------------------+ <-- [x30 / SP] + +// This function takes two arguments: +// * x0: The address of read/write instruction that caused HWASan check fail. +// * x1: The tag size. + +.section .text +.file "hwasan_tag_mismatch_aarch64.S" +.global __hwasan_tag_mismatch +.type __hwasan_tag_mismatch, %function +__hwasan_tag_mismatch: + CFI_STARTPROC + + // Set the CFA to be the return address for caller of __hwasan_check_*. Note + // that we do not emit CFI predicates to describe the contents of this stack + // frame, as this proxy entry point should never be debugged. The contents + // are static and are handled by the unwinder after calling + // __hwasan_tag_mismatch. The frame pointer is already correctly setup + // by __hwasan_check_*. + add x29, sp, #232 + CFI_DEF_CFA(w29, 24) + CFI_OFFSET(w30, -16) + CFI_OFFSET(w29, -24) + + // Save the rest of the registers into the preallocated space left by + // __hwasan_check. + str x28, [sp, #224] + stp x26, x27, [sp, #208] + stp x24, x25, [sp, #192] + stp x22, x23, [sp, #176] + stp x20, x21, [sp, #160] + stp x18, x19, [sp, #144] + stp x16, x17, [sp, #128] + stp x14, x15, [sp, #112] + stp x12, x13, [sp, #96] + stp x10, x11, [sp, #80] + stp x8, x9, [sp, #64] + stp x6, x7, [sp, #48] + stp x4, x5, [sp, #32] + stp x2, x3, [sp, #16] + + // Pass the address of the frame to __hwasan_tag_mismatch_stub, so that it can + // extract the saved registers from this frame without having to worry about + // finding this frame. + mov x2, sp + + bl __hwasan_tag_mismatch_stub + CFI_ENDPROC + +.Lfunc_end0: + .size __hwasan_tag_mismatch, .Lfunc_end0-__hwasan_tag_mismatch + +#endif // defined(__aarch64__) + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_thread.cc b/lib/hwasan/hwasan_thread.cpp index 631c2813eb06..cabf614c005c 100644 --- a/lib/hwasan/hwasan_thread.cc +++ b/lib/hwasan/hwasan_thread.cpp @@ -25,10 +25,18 @@ static u32 RandomSeed() { return seed; } +void Thread::InitRandomState() { + random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; + + // Push a random number of zeros onto the ring buffer so that the first stack + // tag base will be random. + for (tag_t i = 0, e = GenerateRandomTag(); i != e; ++i) + stack_allocations_->push(0); +} + void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { static u64 unique_id; unique_id_ = unique_id++; - random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; if (auto sz = flags()->heap_history_size) heap_allocations_ = HeapAllocationsRingBuffer::New(sz); diff --git a/lib/hwasan/hwasan_thread.h b/lib/hwasan/hwasan_thread.h index 4830473f4aab..42c1e9e124b0 100644 --- a/lib/hwasan/hwasan_thread.h +++ b/lib/hwasan/hwasan_thread.h @@ -1,9 +1,8 @@ //===-- hwasan_thread.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer; class Thread { public: void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + void InitRandomState(); void Destroy(); uptr stack_top() { return stack_top_; } @@ -46,10 +46,6 @@ class Thread { void EnterSymbolizer() { in_symbolizer_++; } void LeaveSymbolizer() { in_symbolizer_--; } - bool InInterceptorScope() { return in_interceptor_scope_; } - void EnterInterceptorScope() { in_interceptor_scope_++; } - void LeaveInterceptorScope() { in_interceptor_scope_--; } - AllocatorCache *allocator_cache() { return &allocator_cache_; } HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; } StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; } @@ -67,11 +63,14 @@ class Thread { Print("Thread: "); } + uptr &vfork_spill() { return vfork_spill_; } + private: // NOTE: There is no Thread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. void ClearShadowForThreadStackAndTLS(); void Print(const char *prefix); + uptr vfork_spill_; uptr stack_top_; uptr stack_bottom_; uptr tls_begin_; @@ -79,7 +78,6 @@ class Thread { unsigned in_signal_handler_; unsigned in_symbolizer_; - unsigned in_interceptor_scope_; u32 random_state_; u32 random_buffer_; diff --git a/lib/hwasan/hwasan_thread_list.cc b/lib/hwasan/hwasan_thread_list.cpp index a31eee84ed93..a31eee84ed93 100644 --- a/lib/hwasan/hwasan_thread_list.cc +++ b/lib/hwasan/hwasan_thread_list.cpp diff --git a/lib/hwasan/hwasan_thread_list.h b/lib/hwasan/hwasan_thread_list.h index 53747b51fd65..914b632d9776 100644 --- a/lib/hwasan/hwasan_thread_list.h +++ b/lib/hwasan/hwasan_thread_list.h @@ -1,9 +1,8 @@ //===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -109,38 +108,52 @@ struct ThreadStats { class HwasanThreadList { public: HwasanThreadList(uptr storage, uptr size) - : free_space_(storage), - free_space_end_(storage + size), - ring_buffer_size_(RingBufferSize()) {} + : free_space_(storage), free_space_end_(storage + size) { + // [storage, storage + size) is used as a vector of + // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements. + // Each element contains + // * a ring buffer at offset 0, + // * a Thread object at offset ring_buffer_size_. + ring_buffer_size_ = RingBufferSize(); + thread_alloc_size_ = + RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); + } Thread *CreateCurrentThread() { Thread *t; { SpinMutexLock l(&list_mutex_); t = free_list_.Pop(); - if (t) - internal_memset((void *)t, 0, sizeof(Thread) + ring_buffer_size_); - else + if (t) { + uptr start = (uptr)t - ring_buffer_size_; + internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread)); + } else { t = AllocThread(); + } live_list_.Push(t); } - t->Init((uptr)(t + 1), ring_buffer_size_); + t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_); AddThreadStats(t); return t; } + void DontNeedThread(Thread *t) { + uptr start = (uptr)t - ring_buffer_size_; + ReleaseMemoryPagesToOS(start, start + thread_alloc_size_); + } + void ReleaseThread(Thread *t) { - // FIXME: madvise away the ring buffer? RemoveThreadStats(t); t->Destroy(); SpinMutexLock l(&list_mutex_); live_list_.Remove(t); free_list_.Push(t); + DontNeedThread(t); } Thread *GetThreadByBufferAddress(uptr p) { - uptr align = ring_buffer_size_ * 2; - return (Thread *)(RoundDownTo(p, align) - sizeof(Thread)); + return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) + + ring_buffer_size_); } uptr MemoryUsedPerThread() { @@ -176,15 +189,17 @@ class HwasanThreadList { private: Thread *AllocThread() { uptr align = ring_buffer_size_ * 2; - uptr ring_buffer_start = RoundUpTo(free_space_ + sizeof(Thread), align); - free_space_ = ring_buffer_start + ring_buffer_size_; + CHECK(IsAligned(free_space_, align)); + Thread *t = (Thread *)(free_space_ + ring_buffer_size_); + free_space_ += thread_alloc_size_; CHECK(free_space_ <= free_space_end_ && "out of thread memory"); - return (Thread *)(ring_buffer_start - sizeof(Thread)); + return t; } uptr free_space_; uptr free_space_end_; uptr ring_buffer_size_; + uptr thread_alloc_size_; ThreadListHead free_list_; ThreadListHead live_list_; |