diff options
Diffstat (limited to 'lib')
1002 files changed, 24867 insertions, 17844 deletions
diff --git a/lib/asan/asan_activation.cc b/lib/asan/asan_activation.cc index d642be93488d..fc97cbb554d0 100644 --- a/lib/asan/asan_activation.cc +++ b/lib/asan/asan_activation.cc @@ -1,9 +1,8 @@ //===-- asan_activation.cc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_activation.h b/lib/asan/asan_activation.h index d5e1ce433001..93c290c2ae2f 100644 --- a/lib/asan/asan_activation.h +++ b/lib/asan/asan_activation.h @@ -1,9 +1,8 @@ //===-- asan_activation.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_activation_flags.inc b/lib/asan/asan_activation_flags.inc index 1c66e5bb53a5..e0fdffc82ac3 100644 --- a/lib/asan/asan_activation_flags.inc +++ b/lib/asan/asan_activation_flags.inc @@ -1,9 +1,8 @@ //===-- asan_activation_flags.inc -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc index c0fad4fa042b..2ca6220d8fd1 100644 --- a/lib/asan/asan_allocator.cc +++ b/lib/asan/asan_allocator.cc @@ -1,9 +1,8 @@ //===-- asan_allocator.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -880,6 +879,17 @@ void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { return SetErrnoOnNull(instance.Calloc(nmemb, size, stack)); } +void *asan_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return asan_realloc(p, nmemb * size, stack); +} + void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { if (!p) return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true)); diff --git a/lib/asan/asan_allocator.h b/lib/asan/asan_allocator.h index c9b37dc7a6d4..6add47be2c9c 100644 --- a/lib/asan/asan_allocator.h +++ b/lib/asan/asan_allocator.h @@ -1,9 +1,8 @@ //===-- asan_allocator.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -134,11 +133,15 @@ const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x2000000000ULL; // 128G. typedef VeryCompactSizeClassMap SizeClassMap; # elif defined(__aarch64__) -// AArch64/SANITIZER_CAN_USER_ALLOCATOR64 is only for 42-bit VMA +// AArch64/SANITIZER_CAN_USE_ALLOCATOR64 is only for 42-bit VMA // so no need to different values for different VMA. const uptr kAllocatorSpace = 0x10000000000ULL; const uptr kAllocatorSize = 0x10000000000ULL; // 3T. typedef DefaultSizeClassMap SizeClassMap; +#elif defined(__sparc__) +const uptr kAllocatorSpace = ~(uptr)0; +const uptr kAllocatorSize = 0x20000000000ULL; // 2T. +typedef DefaultSizeClassMap SizeClassMap; # elif SANITIZER_WINDOWS const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x8000000000ULL; // 500G @@ -163,16 +166,6 @@ template <typename AddressSpaceView> using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>; using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #else // Fallback to SizeClassAllocator32. -static const uptr kRegionSizeLog = 20; -static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -# if SANITIZER_WORDSIZE == 32 -template <typename AddressSpaceView> -using ByteMapASVT = FlatByteMap<kNumRegions, AddressSpaceView>; -# elif SANITIZER_WORDSIZE == 64 -template <typename AddressSpaceView> -using ByteMapASVT = - TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; -# endif typedef CompactSizeClassMap SizeClassMap; template <typename AddressSpaceViewTy> struct AP32 { @@ -180,9 +173,8 @@ struct AP32 { static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 16; typedef __asan::SizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __asan::kRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = AddressSpaceViewTy; - using ByteMap = __asan::ByteMapASVT<AddressSpaceView>; typedef AsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -192,21 +184,12 @@ using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #endif // SANITIZER_CAN_USE_ALLOCATOR64 static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; -template <typename AddressSpaceView> -using AllocatorCacheASVT = - SizeClassAllocatorLocalCache<PrimaryAllocatorASVT<AddressSpaceView>>; -using AllocatorCache = AllocatorCacheASVT<LocalAddressSpaceView>; template <typename AddressSpaceView> -using SecondaryAllocatorASVT = - LargeMmapAllocator<AsanMapUnmapCallback, DefaultLargeMmapAllocatorPtrArray, - AddressSpaceView>; -template <typename AddressSpaceView> using AsanAllocatorASVT = - CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>, - AllocatorCacheASVT<AddressSpaceView>, - SecondaryAllocatorASVT<AddressSpaceView>>; + CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>; using AsanAllocator = AsanAllocatorASVT<LocalAddressSpaceView>; +using AllocatorCache = AsanAllocator::AllocatorCache; struct AsanThreadLocalMallocStorage { uptr quarantine_cache[16]; @@ -226,6 +209,8 @@ void asan_delete(void *ptr, uptr size, uptr alignment, void *asan_malloc(uptr size, BufferedStackTrace *stack); void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack); void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack); +void *asan_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack); void *asan_valloc(uptr size, BufferedStackTrace *stack); void *asan_pvalloc(uptr size, BufferedStackTrace *stack); diff --git a/lib/asan/asan_debugging.cc b/lib/asan/asan_debugging.cc index 7c877ded30cf..7052a371e676 100644 --- a/lib/asan/asan_debugging.cc +++ b/lib/asan/asan_debugging.cc @@ -1,9 +1,8 @@ //===-- asan_debugging.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_descriptions.cc b/lib/asan/asan_descriptions.cc index cdb562d975ff..9b1217a86652 100644 --- a/lib/asan/asan_descriptions.cc +++ b/lib/asan/asan_descriptions.cc @@ -1,9 +1,8 @@ //===-- asan_descriptions.cc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_descriptions.h b/lib/asan/asan_descriptions.h index 5c2d7662a35a..0226d844afc9 100644 --- a/lib/asan/asan_descriptions.h +++ b/lib/asan/asan_descriptions.h @@ -1,9 +1,8 @@ //===-- asan_descriptions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_errors.cc b/lib/asan/asan_errors.cc index 0ecd30dcacfd..d598e37b940e 100644 --- a/lib/asan/asan_errors.cc +++ b/lib/asan/asan_errors.cc @@ -1,9 +1,8 @@ //===-- asan_errors.cc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,7 +35,7 @@ static void OnStackUnwind(const SignalContext &sig, // corresponding code in the sanitizer_common and we use this callback to // print it. static_cast<const ScarinessScoreBase *>(callback_context)->Print(); - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, fast); + stack->Unwind(sig.pc, sig.bp, sig.context, fast); } void ErrorDeadlySignal::Print() { @@ -178,6 +177,19 @@ void ErrorCallocOverflow::Print() { ReportErrorSummary(scariness.GetDescription(), stack); } +void ErrorReallocArrayOverflow::Print() { + Decorator d; + Printf("%s", d.Error()); + Report( + "ERROR: AddressSanitizer: reallocarray parameters overflow: count * size " + "(%zd * %zd) cannot be represented in type size_t (thread %s)\n", + count, size, AsanThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(scariness.GetDescription(), stack); +} + void ErrorPvallocOverflow::Print() { Decorator d; Printf("%s", d.Error()); diff --git a/lib/asan/asan_errors.h b/lib/asan/asan_errors.h index 7ddd7e94e0fc..b84f56c18535 100644 --- a/lib/asan/asan_errors.h +++ b/lib/asan/asan_errors.h @@ -1,9 +1,8 @@ //===-- asan_errors.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -164,6 +163,21 @@ struct ErrorCallocOverflow : ErrorBase { void Print(); }; +struct ErrorReallocArrayOverflow : ErrorBase { + const BufferedStackTrace *stack; + uptr count; + uptr size; + + ErrorReallocArrayOverflow() = default; // (*) + ErrorReallocArrayOverflow(u32 tid, BufferedStackTrace *stack_, uptr count_, + uptr size_) + : ErrorBase(tid, 10, "reallocarray-overflow"), + stack(stack_), + count(count_), + size(size_) {} + void Print(); +}; + struct ErrorPvallocOverflow : ErrorBase { const BufferedStackTrace *stack; uptr size; @@ -372,6 +386,7 @@ struct ErrorGeneric : ErrorBase { macro(MallocUsableSizeNotOwned) \ macro(SanitizerGetAllocatedSizeNotOwned) \ macro(CallocOverflow) \ + macro(ReallocArrayOverflow) \ macro(PvallocOverflow) \ macro(InvalidAllocationAlignment) \ macro(InvalidAlignedAllocAlignment) \ @@ -389,8 +404,10 @@ struct ErrorGeneric : ErrorBase { #define ASAN_DEFINE_ERROR_KIND(name) kErrorKind##name, #define ASAN_ERROR_DESCRIPTION_MEMBER(name) Error##name name; -#define ASAN_ERROR_DESCRIPTION_CONSTRUCTOR(name) \ - ErrorDescription(Error##name const &e) : kind(kErrorKind##name), name(e) {} +#define ASAN_ERROR_DESCRIPTION_CONSTRUCTOR(name) \ + ErrorDescription(Error##name const &e) : kind(kErrorKind##name) { \ + internal_memcpy(&name, &e, sizeof(name)); \ + } #define ASAN_ERROR_DESCRIPTION_PRINT(name) \ case kErrorKind##name: \ return name.Print(); diff --git a/lib/asan/asan_fake_stack.cc b/lib/asan/asan_fake_stack.cc index 1c6184e3c7fc..f8e1ac4b7bfe 100644 --- a/lib/asan/asan_fake_stack.cc +++ b/lib/asan/asan_fake_stack.cc @@ -1,9 +1,8 @@ //===-- asan_fake_stack.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_fake_stack.h b/lib/asan/asan_fake_stack.h index da9a91c23025..59ba85218f88 100644 --- a/lib/asan/asan_fake_stack.h +++ b/lib/asan/asan_fake_stack.h @@ -1,9 +1,8 @@ //===-- asan_fake_stack.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_flags.cc b/lib/asan/asan_flags.cc index 5682ab4eb476..89e98936129d 100644 --- a/lib/asan/asan_flags.cc +++ b/lib/asan/asan_flags.cc @@ -1,9 +1,8 @@ //===-- asan_flags.cc -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -121,12 +120,12 @@ void InitializeFlags() { #endif // Override from command line. - asan_parser.ParseString(GetEnv("ASAN_OPTIONS")); + asan_parser.ParseStringFromEnv("ASAN_OPTIONS"); #if CAN_SANITIZE_LEAKS - lsan_parser.ParseString(GetEnv("LSAN_OPTIONS")); + lsan_parser.ParseStringFromEnv("LSAN_OPTIONS"); #endif #if CAN_SANITIZE_UB - ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); #endif InitializeCommonFlags(); diff --git a/lib/asan/asan_flags.h b/lib/asan/asan_flags.h index 4935161c30f1..b55c81f07d4b 100644 --- a/lib/asan/asan_flags.h +++ b/lib/asan/asan_flags.h @@ -1,9 +1,8 @@ //===-- asan_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_flags.inc b/lib/asan/asan_flags.inc index a9c97d53b31f..d360e03ca55e 100644 --- a/lib/asan/asan_flags.inc +++ b/lib/asan/asan_flags.inc @@ -1,9 +1,8 @@ //===-- asan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -159,3 +158,5 @@ ASAN_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true, ASAN_FLAG(bool, verify_asan_link_order, true, "Check position of ASan runtime in library list (needs to be disabled" " when other library has to be preloaded system-wide)") +ASAN_FLAG(bool, windows_hook_rtl_allocators, false, + "(Windows only) enable hooking of Rtl(Allocate|Free|Size|ReAllocate)Heap.") diff --git a/lib/asan/asan_fuchsia.cc b/lib/asan/asan_fuchsia.cc index 34399c92310b..aebc17f38b4b 100644 --- a/lib/asan/asan_fuchsia.cc +++ b/lib/asan/asan_fuchsia.cc @@ -1,9 +1,8 @@ //===-- asan_fuchsia.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // @@ -179,7 +178,7 @@ static void ThreadStartHook(void *hook, uptr os_id) { SetCurrentThread(thread); // In lieu of AsanThread::ThreadStart. - asanThreadRegistry().StartThread(thread->tid(), os_id, /*workerthread*/ false, + asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular, nullptr); } diff --git a/lib/asan/asan_globals.cc b/lib/asan/asan_globals.cc index 146234ac6c64..a831fdf9cd93 100644 --- a/lib/asan/asan_globals.cc +++ b/lib/asan/asan_globals.cc @@ -1,9 +1,8 @@ //===-- asan_globals.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -116,11 +115,12 @@ int GetGlobalsForAddress(uptr addr, Global *globals, u32 *reg_sites, if (flags()->report_globals >= 2) ReportGlobal(g, "Search"); if (IsAddressNearGlobal(addr, g)) { - globals[res] = g; + internal_memcpy(&globals[res], &g, sizeof(g)); if (reg_sites) reg_sites[res] = FindRegistrationSite(&g); res++; - if (res == max_globals) break; + if (res == max_globals) + break; } } return res; diff --git a/lib/asan/asan_globals_win.cc b/lib/asan/asan_globals_win.cc index 0e75992bfc8d..bdce37f701e1 100644 --- a/lib/asan/asan_globals_win.cc +++ b/lib/asan/asan_globals_win.cc @@ -1,9 +1,8 @@ //===-- asan_globals_win.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_init_version.h b/lib/asan/asan_init_version.h index c49fcd740248..b806d794e056 100644 --- a/lib/asan/asan_init_version.h +++ b/lib/asan/asan_init_version.h @@ -1,9 +1,8 @@ //===-- asan_init_version.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_interceptors.cc b/lib/asan/asan_interceptors.cc index aac2bb8a6bbf..7eea7d2f942b 100644 --- a/lib/asan/asan_interceptors.cc +++ b/lib/asan/asan_interceptors.cc @@ -1,9 +1,8 @@ //===-- asan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -580,6 +579,11 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, } #endif // ASAN_INTERCEPT___CXA_ATEXIT +#if ASAN_INTERCEPT_VFORK +DEFINE_REAL(int, vfork) +DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork) +#endif + // ---------------------- InitializeAsanInterceptors ---------------- {{{1 namespace __asan { void InitializeAsanInterceptors() { @@ -657,6 +661,10 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(__cxa_atexit); #endif +#if ASAN_INTERCEPT_VFORK + ASAN_INTERCEPT_FUNC(vfork); +#endif + InitializePlatformInterceptors(); VReport(1, "AddressSanitizer: libc interceptors initialized\n"); diff --git a/lib/asan/asan_interceptors.h b/lib/asan/asan_interceptors.h index 50895b167990..381b03984191 100644 --- a/lib/asan/asan_interceptors.h +++ b/lib/asan/asan_interceptors.h @@ -1,9 +1,8 @@ //===-- asan_interceptors.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -106,6 +105,13 @@ void InitializePlatformInterceptors(); # define ASAN_INTERCEPT___STRDUP 0 #endif +#if SANITIZER_LINUX && (defined(__arm__) || defined(__aarch64__) || \ + defined(__i386__) || defined(__x86_64__)) +# define ASAN_INTERCEPT_VFORK 1 +#else +# define ASAN_INTERCEPT_VFORK 0 +#endif + DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size) DECLARE_REAL(char*, strchr, const char *str, int c) DECLARE_REAL(SIZE_T, strlen, const char *s) @@ -114,16 +120,16 @@ DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen) DECLARE_REAL(char*, strstr, const char *s1, const char *s2) #if !SANITIZER_MAC -#define ASAN_INTERCEPT_FUNC(name) \ - do { \ - if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ - VReport(1, "AddressSanitizer: failed to intercept '" #name "'\n"); \ +#define ASAN_INTERCEPT_FUNC(name) \ + do { \ + if (!INTERCEPT_FUNCTION(name)) \ + VReport(1, "AddressSanitizer: failed to intercept '%s'\n'", #name); \ } while (0) -#define ASAN_INTERCEPT_FUNC_VER(name, ver) \ - do { \ - if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ - VReport( \ - 1, "AddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ +#define ASAN_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if (!INTERCEPT_FUNCTION_VER(name, ver)) \ + VReport(1, "AddressSanitizer: failed to intercept '%s@@%s'\n", #name, \ + #ver); \ } while (0) #else // OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION. diff --git a/lib/asan/asan_interceptors_memintrinsics.cc b/lib/asan/asan_interceptors_memintrinsics.cc index 39e32cdad12e..e17f9ba4aab5 100644 --- a/lib/asan/asan_interceptors_memintrinsics.cc +++ b/lib/asan/asan_interceptors_memintrinsics.cc @@ -1,9 +1,8 @@ //===-- asan_interceptors_memintrinsics.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/asan/asan_interceptors_memintrinsics.h b/lib/asan/asan_interceptors_memintrinsics.h index a071e8f684a0..1fd65fe24953 100644 --- a/lib/asan/asan_interceptors_memintrinsics.h +++ b/lib/asan/asan_interceptors_memintrinsics.h @@ -1,9 +1,8 @@ //===-- asan_interceptors_memintrinsics.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/asan/asan_interceptors_vfork.S b/lib/asan/asan_interceptors_vfork.S new file mode 100644 index 000000000000..90a169d4b609 --- /dev/null +++ b/lib/asan/asan_interceptors_vfork.S @@ -0,0 +1,12 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__linux__) +#define COMMON_INTERCEPTOR_SPILL_AREA __asan_extra_spill_area +#define COMMON_INTERCEPTOR_HANDLE_VFORK __asan_handle_vfork +#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S" +#endif + +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/asan/asan_interface.inc b/lib/asan/asan_interface.inc index e65f61722b10..7c341f22e15f 100644 --- a/lib/asan/asan_interface.inc +++ b/lib/asan/asan_interface.inc @@ -1,9 +1,8 @@ //===-- asan_interface.inc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Asan interface list. @@ -39,6 +38,7 @@ INTERFACE_FUNCTION(__asan_get_report_pc) INTERFACE_FUNCTION(__asan_get_report_sp) INTERFACE_FUNCTION(__asan_get_shadow_mapping) INTERFACE_FUNCTION(__asan_handle_no_return) +INTERFACE_FUNCTION(__asan_handle_vfork) INTERFACE_FUNCTION(__asan_init) INTERFACE_FUNCTION(__asan_load_cxx_array_cookie) INTERFACE_FUNCTION(__asan_load1) diff --git a/lib/asan/asan_interface_internal.h b/lib/asan/asan_interface_internal.h index b974c0cc4b43..c83aa11d741a 100644 --- a/lib/asan/asan_interface_internal.h +++ b/lib/asan/asan_interface_internal.h @@ -1,9 +1,8 @@ //===-- asan_interface_internal.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -250,6 +249,8 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char* __asan_default_suppressions(); + + SANITIZER_INTERFACE_ATTRIBUTE void __asan_handle_vfork(void *sp); } // extern "C" #endif // ASAN_INTERFACE_INTERNAL_H diff --git a/lib/asan/asan_internal.h b/lib/asan/asan_internal.h index 57869497c7d3..e4f771079293 100644 --- a/lib/asan/asan_internal.h +++ b/lib/asan/asan_internal.h @@ -1,9 +1,8 @@ //===-- asan_internal.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_linux.cc b/lib/asan/asan_linux.cc index a150b1955d60..f9182328916f 100644 --- a/lib/asan/asan_linux.cc +++ b/lib/asan/asan_linux.cc @@ -1,9 +1,8 @@ //===-- asan_linux.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_mac.cc b/lib/asan/asan_mac.cc index 17a0ec57701d..e776acd2f539 100644 --- a/lib/asan/asan_mac.cc +++ b/lib/asan/asan_mac.cc @@ -1,9 +1,8 @@ //===-- asan_mac.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -182,8 +181,8 @@ void asan_register_worker_thread(int parent_tid, StackTrace *stack) { t = AsanThread::Create(/* start_routine */ nullptr, /* arg */ nullptr, parent_tid, stack, /* detached */ true); t->Init(); - asanThreadRegistry().StartThread(t->tid(), GetTid(), - /* workerthread */ true, 0); + asanThreadRegistry().StartThread(t->tid(), GetTid(), ThreadType::Worker, + nullptr); SetCurrentThread(t); } } diff --git a/lib/asan/asan_malloc_linux.cc b/lib/asan/asan_malloc_linux.cc index 0a534fe21168..86fcd3b12d58 100644 --- a/lib/asan/asan_malloc_linux.cc +++ b/lib/asan/asan_malloc_linux.cc @@ -1,9 +1,8 @@ //===-- asan_malloc_linux.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -166,6 +165,14 @@ INTERCEPTOR(void*, realloc, void *ptr, uptr size) { return asan_realloc(ptr, size, &stack); } +#if SANITIZER_INTERCEPT_REALLOCARRAY +INTERCEPTOR(void*, reallocarray, void *ptr, uptr nmemb, uptr size) { + ENSURE_ASAN_INITED(); + GET_STACK_TRACE_MALLOC; + return asan_reallocarray(ptr, nmemb, size, &stack); +} +#endif // SANITIZER_INTERCEPT_REALLOCARRAY + #if SANITIZER_INTERCEPT_MEMALIGN INTERCEPTOR(void*, memalign, uptr boundary, uptr size) { GET_STACK_TRACE_MALLOC; diff --git a/lib/asan/asan_malloc_local.h b/lib/asan/asan_malloc_local.h index 0e8de207d98d..3f784b90c739 100644 --- a/lib/asan/asan_malloc_local.h +++ b/lib/asan/asan_malloc_local.h @@ -1,9 +1,8 @@ //===-- asan_malloc_local.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,25 +17,34 @@ #include "sanitizer_common/sanitizer_platform.h" #include "asan_internal.h" -// On RTEMS, we use the local pool to handle memory allocation when the ASan -// run-time is not up. static INLINE bool EarlyMalloc() { - return SANITIZER_RTEMS && (!__asan::asan_inited || - __asan::asan_init_is_running); + return SANITIZER_RTEMS && + (!__asan::asan_inited || __asan::asan_init_is_running); } -void* MemalignFromLocalPool(uptr alignment, uptr size); - #if SANITIZER_RTEMS bool IsFromLocalPool(const void *ptr); +void *MemalignFromLocalPool(uptr alignment, uptr size); + +// On RTEMS, we use the local pool to handle memory allocation when the ASan +// run-time is not up. This macro is expanded in the context of the operator new +// implementation. +#define MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow) \ + do { \ + if (UNLIKELY(EarlyMalloc())) { \ + void *res = MemalignFromLocalPool(SHADOW_GRANULARITY, size); \ + if (!nothrow) \ + CHECK(res); \ + return res; \ + } \ + } while (0) -#define ALLOCATE_FROM_LOCAL_POOL UNLIKELY(EarlyMalloc()) #define IS_FROM_LOCAL_POOL(ptr) UNLIKELY(IsFromLocalPool(ptr)) #else // SANITIZER_RTEMS -#define ALLOCATE_FROM_LOCAL_POOL 0 +#define MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow) #define IS_FROM_LOCAL_POOL(ptr) 0 #endif // SANITIZER_RTEMS diff --git a/lib/asan/asan_malloc_mac.cc b/lib/asan/asan_malloc_mac.cc index 27281f1bc834..06dc1c289267 100644 --- a/lib/asan/asan_malloc_mac.cc +++ b/lib/asan/asan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- asan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #include "asan_report.h" #include "asan_stack.h" #include "asan_stats.h" +#include "lsan/lsan_common.h" using namespace __asan; #define COMMON_MALLOC_ZONE_NAME "asan" @@ -58,10 +58,13 @@ using namespace __asan; GET_STACK_TRACE_FREE; \ ReportMacMzReallocUnknown((uptr)ptr, (uptr)zone_ptr, zone_name, &stack); #define COMMON_MALLOC_NAMESPACE __asan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 1 #include "sanitizer_common/sanitizer_malloc_mac.inc" namespace COMMON_MALLOC_NAMESPACE { + bool HandleDlopenInit() { static_assert(SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be true"); @@ -82,4 +85,18 @@ bool HandleDlopenInit() { } } // namespace COMMON_MALLOC_NAMESPACE +namespace { + +void mi_extra_init(sanitizer_malloc_introspection_t *mi) { + uptr last_byte_plus_one = 0; + mi->allocator_ptr = 0; + // Range is [begin_ptr, end_ptr) + __lsan::GetAllocatorGlobalRange(&(mi->allocator_ptr), &last_byte_plus_one); + CHECK_NE(mi->allocator_ptr, 0); + CHECK_GT(last_byte_plus_one, mi->allocator_ptr); + mi->allocator_size = last_byte_plus_one - (mi->allocator_ptr); + CHECK_GT(mi->allocator_size, 0); +} +} // namespace + #endif diff --git a/lib/asan/asan_malloc_win.cc b/lib/asan/asan_malloc_win.cc index 887936431931..5fad55d6e284 100644 --- a/lib/asan/asan_malloc_win.cc +++ b/lib/asan/asan_malloc_win.cc @@ -1,9 +1,8 @@ //===-- asan_malloc_win.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,8 +11,16 @@ // Windows-specific malloc interception. //===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_WINDOWS +#include "asan_allocator.h" +#include "asan_interceptors.h" +#include "asan_internal.h" +#include "asan_stack.h" +#include "interception/interception.h" +#include <stddef.h> + // Intentionally not including windows.h here, to avoid the risk of // pulling in conflicting declarations of these functions. (With mingw-w64, // there's a risk of windows.h pulling in stdint.h.) @@ -22,17 +29,30 @@ typedef void *HANDLE; typedef const void *LPCVOID; typedef void *LPVOID; -#define HEAP_ZERO_MEMORY 0x00000008 -#define HEAP_REALLOC_IN_PLACE_ONLY 0x00000010 +typedef unsigned long DWORD; +constexpr unsigned long HEAP_ZERO_MEMORY = 0x00000008; +constexpr unsigned long HEAP_REALLOC_IN_PLACE_ONLY = 0x00000010; +constexpr unsigned long HEAP_ALLOCATE_SUPPORTED_FLAGS = (HEAP_ZERO_MEMORY); +constexpr unsigned long HEAP_ALLOCATE_UNSUPPORTED_FLAGS = + (~HEAP_ALLOCATE_SUPPORTED_FLAGS); +constexpr unsigned long HEAP_FREE_SUPPORTED_FLAGS = (0); +constexpr unsigned long HEAP_FREE_UNSUPPORTED_FLAGS = + (~HEAP_ALLOCATE_SUPPORTED_FLAGS); +constexpr unsigned long HEAP_REALLOC_SUPPORTED_FLAGS = + (HEAP_REALLOC_IN_PLACE_ONLY | HEAP_ZERO_MEMORY); +constexpr unsigned long HEAP_REALLOC_UNSUPPORTED_FLAGS = + (~HEAP_ALLOCATE_SUPPORTED_FLAGS); -#include "asan_allocator.h" -#include "asan_interceptors.h" -#include "asan_internal.h" -#include "asan_stack.h" -#include "interception/interception.h" +extern "C" { +LPVOID WINAPI HeapAlloc(HANDLE hHeap, DWORD dwFlags, size_t dwBytes); +LPVOID WINAPI HeapReAlloc(HANDLE hHeap, DWORD dwFlags, LPVOID lpMem, + size_t dwBytes); +BOOL WINAPI HeapFree(HANDLE hHeap, DWORD dwFlags, LPVOID lpMem); +size_t WINAPI HeapSize(HANDLE hHeap, DWORD dwFlags, LPCVOID lpMem); -#include <stddef.h> +BOOL WINAPI HeapValidate(HANDLE hHeap, DWORD dwFlags, LPCVOID lpMem); +} using namespace __asan; // NOLINT @@ -49,6 +69,18 @@ using namespace __asan; // NOLINT extern "C" { ALLOCATION_FUNCTION_ATTRIBUTE +size_t _msize(void *ptr) { + GET_CURRENT_PC_BP_SP; + (void)sp; + return asan_malloc_usable_size(ptr, pc, bp); +} + +ALLOCATION_FUNCTION_ATTRIBUTE +size_t _msize_base(void *ptr) { + return _msize(ptr); +} + +ALLOCATION_FUNCTION_ATTRIBUTE void free(void *ptr) { GET_STACK_TRACE_FREE; return asan_free(ptr, &stack, FROM_MALLOC); @@ -125,7 +157,16 @@ void *_recalloc(void *p, size_t n, size_t elem_size) { const size_t size = n * elem_size; if (elem_size != 0 && size / elem_size != n) return 0; - return realloc(p, size); + + size_t old_size = _msize(p); + void *new_alloc = malloc(size); + if (new_alloc) { + REAL(memcpy)(new_alloc, p, Min<size_t>(size, old_size)); + if (old_size < size) + REAL(memset)(((u8 *)new_alloc) + old_size, 0, size - old_size); + free(p); + } + return new_alloc; } ALLOCATION_FUNCTION_ATTRIBUTE @@ -134,18 +175,6 @@ void *_recalloc_base(void *p, size_t n, size_t elem_size) { } ALLOCATION_FUNCTION_ATTRIBUTE -size_t _msize(void *ptr) { - GET_CURRENT_PC_BP_SP; - (void)sp; - return asan_malloc_usable_size(ptr, pc, bp); -} - -ALLOCATION_FUNCTION_ATTRIBUTE -size_t _msize_base(void *ptr) { - return _msize(ptr); -} - -ALLOCATION_FUNCTION_ATTRIBUTE void *_expand(void *memblock, size_t size) { // _expand is used in realloc-like functions to resize the buffer if possible. // We don't want memory to stand still while resizing buffers, so return 0. @@ -175,43 +204,280 @@ int _CrtSetReportMode(int, int) { } } // extern "C" +#define OWNED_BY_RTL(heap, memory) \ + (!__sanitizer_get_ownership(memory) && HeapValidate(heap, 0, memory)) + +INTERCEPTOR_WINAPI(size_t, HeapSize, HANDLE hHeap, DWORD dwFlags, + LPCVOID lpMem) { + // If the RTL allocators are hooked we need to check whether the ASAN + // allocator owns the pointer we're about to use. Allocations occur before + // interception takes place, so if it is not owned by the RTL heap we can + // pass it to the ASAN heap for inspection. + if (flags()->windows_hook_rtl_allocators) { + if (!asan_inited || OWNED_BY_RTL(hHeap, lpMem)) + return REAL(HeapSize)(hHeap, dwFlags, lpMem); + } else { + CHECK(dwFlags == 0 && "unsupported heap flags"); + } + GET_CURRENT_PC_BP_SP; + (void)sp; + return asan_malloc_usable_size(lpMem, pc, bp); +} + INTERCEPTOR_WINAPI(LPVOID, HeapAlloc, HANDLE hHeap, DWORD dwFlags, - SIZE_T dwBytes) { + size_t dwBytes) { + // If the ASAN runtime is not initialized, or we encounter an unsupported + // flag, fall back to the original allocator. + if (flags()->windows_hook_rtl_allocators) { + if (UNLIKELY(!asan_inited || + (dwFlags & HEAP_ALLOCATE_UNSUPPORTED_FLAGS) != 0)) { + return REAL(HeapAlloc)(hHeap, dwFlags, dwBytes); + } + } else { + // In the case that we don't hook the rtl allocators, + // this becomes an assert since there is no failover to the original + // allocator. + CHECK((HEAP_ALLOCATE_UNSUPPORTED_FLAGS & dwFlags) != 0 && + "unsupported flags"); + } GET_STACK_TRACE_MALLOC; void *p = asan_malloc(dwBytes, &stack); // Reading MSDN suggests that the *entire* usable allocation is zeroed out. // Otherwise it is difficult to HeapReAlloc with HEAP_ZERO_MEMORY. // https://blogs.msdn.microsoft.com/oldnewthing/20120316-00/?p=8083 - if (dwFlags == HEAP_ZERO_MEMORY) - internal_memset(p, 0, asan_mz_size(p)); - else - CHECK(dwFlags == 0 && "unsupported heap flags"); + if (p && (dwFlags & HEAP_ZERO_MEMORY)) { + GET_CURRENT_PC_BP_SP; + (void)sp; + auto usable_size = asan_malloc_usable_size(p, pc, bp); + internal_memset(p, 0, usable_size); + } return p; } INTERCEPTOR_WINAPI(BOOL, HeapFree, HANDLE hHeap, DWORD dwFlags, LPVOID lpMem) { - CHECK(dwFlags == 0 && "unsupported heap flags"); + // Heap allocations happen before this function is hooked, so we must fall + // back to the original function if the pointer is not from the ASAN heap, + // or unsupported flags are provided. + if (flags()->windows_hook_rtl_allocators) { + if (OWNED_BY_RTL(hHeap, lpMem)) + return REAL(HeapFree)(hHeap, dwFlags, lpMem); + } else { + CHECK((HEAP_FREE_UNSUPPORTED_FLAGS & dwFlags) != 0 && "unsupported flags"); + } GET_STACK_TRACE_FREE; asan_free(lpMem, &stack, FROM_MALLOC); return true; } -INTERCEPTOR_WINAPI(LPVOID, HeapReAlloc, HANDLE hHeap, DWORD dwFlags, - LPVOID lpMem, SIZE_T dwBytes) { +namespace __asan { +using AllocFunction = LPVOID(WINAPI *)(HANDLE, DWORD, size_t); +using ReAllocFunction = LPVOID(WINAPI *)(HANDLE, DWORD, LPVOID, size_t); +using SizeFunction = size_t(WINAPI *)(HANDLE, DWORD, LPVOID); +using FreeFunction = BOOL(WINAPI *)(HANDLE, DWORD, LPVOID); + +void *SharedReAlloc(ReAllocFunction reallocFunc, SizeFunction heapSizeFunc, + FreeFunction freeFunc, AllocFunction allocFunc, + HANDLE hHeap, DWORD dwFlags, LPVOID lpMem, size_t dwBytes) { + CHECK(reallocFunc && heapSizeFunc && freeFunc && allocFunc); GET_STACK_TRACE_MALLOC; - // Realloc should never reallocate in place. + GET_CURRENT_PC_BP_SP; + (void)sp; + if (flags()->windows_hook_rtl_allocators) { + enum AllocationOwnership { NEITHER = 0, ASAN = 1, RTL = 2 }; + AllocationOwnership ownershipState; + bool owned_rtlalloc = false; + bool owned_asan = __sanitizer_get_ownership(lpMem); + + if (!owned_asan) + owned_rtlalloc = HeapValidate(hHeap, 0, lpMem); + + if (owned_asan && !owned_rtlalloc) + ownershipState = ASAN; + else if (!owned_asan && owned_rtlalloc) + ownershipState = RTL; + else if (!owned_asan && !owned_rtlalloc) + ownershipState = NEITHER; + + // If this heap block which was allocated before the ASAN + // runtime came up, use the real HeapFree function. + if (UNLIKELY(!asan_inited)) { + return reallocFunc(hHeap, dwFlags, lpMem, dwBytes); + } + bool only_asan_supported_flags = + (HEAP_REALLOC_UNSUPPORTED_FLAGS & dwFlags) == 0; + + if (ownershipState == RTL || + (ownershipState == NEITHER && !only_asan_supported_flags)) { + if (only_asan_supported_flags) { + // if this is a conversion to ASAN upported flags, transfer this + // allocation to the ASAN allocator + void *replacement_alloc; + if (dwFlags & HEAP_ZERO_MEMORY) + replacement_alloc = asan_calloc(1, dwBytes, &stack); + else + replacement_alloc = asan_malloc(dwBytes, &stack); + if (replacement_alloc) { + size_t old_size = heapSizeFunc(hHeap, dwFlags, lpMem); + if (old_size == ((size_t)0) - 1) { + asan_free(replacement_alloc, &stack, FROM_MALLOC); + return nullptr; + } + REAL(memcpy)(replacement_alloc, lpMem, old_size); + freeFunc(hHeap, dwFlags, lpMem); + } + return replacement_alloc; + } else { + // owned by rtl or neither with unsupported ASAN flags, + // just pass back to original allocator + CHECK(ownershipState == RTL || ownershipState == NEITHER); + CHECK(!only_asan_supported_flags); + return reallocFunc(hHeap, dwFlags, lpMem, dwBytes); + } + } + + if (ownershipState == ASAN && !only_asan_supported_flags) { + // Conversion to unsupported flags allocation, + // transfer this allocation back to the original allocator. + void *replacement_alloc = allocFunc(hHeap, dwFlags, dwBytes); + size_t old_usable_size = 0; + if (replacement_alloc) { + old_usable_size = asan_malloc_usable_size(lpMem, pc, bp); + REAL(memcpy)(replacement_alloc, lpMem, + Min<size_t>(dwBytes, old_usable_size)); + asan_free(lpMem, &stack, FROM_MALLOC); + } + return replacement_alloc; + } + + CHECK((ownershipState == ASAN || ownershipState == NEITHER) && + only_asan_supported_flags); + // At this point we should either be ASAN owned with ASAN supported flags + // or we owned by neither and have supported flags. + // Pass through even when it's neither since this could be a null realloc or + // UAF that ASAN needs to catch. + } else { + CHECK((HEAP_REALLOC_UNSUPPORTED_FLAGS & dwFlags) != 0 && + "unsupported flags"); + } + // asan_realloc will never reallocate in place, so for now this flag is + // unsupported until we figure out a way to fake this. if (dwFlags & HEAP_REALLOC_IN_PLACE_ONLY) return nullptr; - CHECK(dwFlags == 0 && "unsupported heap flags"); - return asan_realloc(lpMem, dwBytes, &stack); + + // HeapReAlloc and HeapAlloc both happily accept 0 sized allocations. + // passing a 0 size into asan_realloc will free the allocation. + // To avoid this and keep behavior consistent, fudge the size if 0. + // (asan_malloc already does this) + if (dwBytes == 0) + dwBytes = 1; + + size_t old_size; + if (dwFlags & HEAP_ZERO_MEMORY) + old_size = asan_malloc_usable_size(lpMem, pc, bp); + + void *ptr = asan_realloc(lpMem, dwBytes, &stack); + if (ptr == nullptr) + return nullptr; + + if (dwFlags & HEAP_ZERO_MEMORY) { + size_t new_size = asan_malloc_usable_size(ptr, pc, bp); + if (old_size < new_size) + REAL(memset)(((u8 *)ptr) + old_size, 0, new_size - old_size); + } + + return ptr; } +} // namespace __asan -INTERCEPTOR_WINAPI(SIZE_T, HeapSize, HANDLE hHeap, DWORD dwFlags, - LPCVOID lpMem) { - CHECK(dwFlags == 0 && "unsupported heap flags"); +INTERCEPTOR_WINAPI(LPVOID, HeapReAlloc, HANDLE hHeap, DWORD dwFlags, + LPVOID lpMem, size_t dwBytes) { + return SharedReAlloc(REAL(HeapReAlloc), (SizeFunction)REAL(HeapSize), + REAL(HeapFree), REAL(HeapAlloc), hHeap, dwFlags, lpMem, + dwBytes); +} + +// The following functions are undocumented and subject to change. +// However, hooking them is necessary to hook Windows heap +// allocations with detours and their definitions are unlikely to change. +// Comments in /minkernel/ntos/rtl/heappublic.c indicate that these functions +// are part of the heap's public interface. +typedef unsigned long LOGICAL; + +// This function is documented as part of the Driver Development Kit but *not* +// the Windows Development Kit. +LOGICAL RtlFreeHeap(void* HeapHandle, DWORD Flags, + void* BaseAddress); + +// This function is documented as part of the Driver Development Kit but *not* +// the Windows Development Kit. +void* RtlAllocateHeap(void* HeapHandle, DWORD Flags, size_t Size); + +// This function is completely undocumented. +void* +RtlReAllocateHeap(void* HeapHandle, DWORD Flags, void* BaseAddress, + size_t Size); + +// This function is completely undocumented. +size_t RtlSizeHeap(void* HeapHandle, DWORD Flags, void* BaseAddress); + +INTERCEPTOR_WINAPI(size_t, RtlSizeHeap, HANDLE HeapHandle, DWORD Flags, + void* BaseAddress) { + if (!flags()->windows_hook_rtl_allocators || + UNLIKELY(!asan_inited || OWNED_BY_RTL(HeapHandle, BaseAddress))) { + return REAL(RtlSizeHeap)(HeapHandle, Flags, BaseAddress); + } GET_CURRENT_PC_BP_SP; (void)sp; - return asan_malloc_usable_size(lpMem, pc, bp); + return asan_malloc_usable_size(BaseAddress, pc, bp); +} + +INTERCEPTOR_WINAPI(BOOL, RtlFreeHeap, HANDLE HeapHandle, DWORD Flags, + void* BaseAddress) { + // Heap allocations happen before this function is hooked, so we must fall + // back to the original function if the pointer is not from the ASAN heap, or + // unsupported flags are provided. + if (!flags()->windows_hook_rtl_allocators || + UNLIKELY((HEAP_FREE_UNSUPPORTED_FLAGS & Flags) != 0 || + OWNED_BY_RTL(HeapHandle, BaseAddress))) { + return REAL(RtlFreeHeap)(HeapHandle, Flags, BaseAddress); + } + GET_STACK_TRACE_FREE; + asan_free(BaseAddress, &stack, FROM_MALLOC); + return true; +} + +INTERCEPTOR_WINAPI(void*, RtlAllocateHeap, HANDLE HeapHandle, DWORD Flags, + size_t Size) { + // If the ASAN runtime is not initialized, or we encounter an unsupported + // flag, fall back to the original allocator. + if (!flags()->windows_hook_rtl_allocators || + UNLIKELY(!asan_inited || + (Flags & HEAP_ALLOCATE_UNSUPPORTED_FLAGS) != 0)) { + return REAL(RtlAllocateHeap)(HeapHandle, Flags, Size); + } + GET_STACK_TRACE_MALLOC; + void *p; + // Reading MSDN suggests that the *entire* usable allocation is zeroed out. + // Otherwise it is difficult to HeapReAlloc with HEAP_ZERO_MEMORY. + // https://blogs.msdn.microsoft.com/oldnewthing/20120316-00/?p=8083 + if (Flags & HEAP_ZERO_MEMORY) { + p = asan_calloc(Size, 1, &stack); + } else { + p = asan_malloc(Size, &stack); + } + return p; +} + +INTERCEPTOR_WINAPI(void*, RtlReAllocateHeap, HANDLE HeapHandle, DWORD Flags, + void* BaseAddress, size_t Size) { + // If it's actually a heap block which was allocated before the ASAN runtime + // came up, use the real RtlFreeHeap function. + if (!flags()->windows_hook_rtl_allocators) + return REAL(RtlReAllocateHeap)(HeapHandle, Flags, BaseAddress, Size); + + return SharedReAlloc(REAL(RtlReAllocateHeap), REAL(RtlSizeHeap), + REAL(RtlFreeHeap), REAL(RtlAllocateHeap), HeapHandle, + Flags, BaseAddress, Size); } namespace __asan { @@ -244,6 +510,34 @@ void ReplaceSystemMalloc() { TryToOverrideFunction("_expand", (uptr)_expand); TryToOverrideFunction("_expand_base", (uptr)_expand); + if (flags()->windows_hook_rtl_allocators) { + INTERCEPT_FUNCTION(HeapSize); + INTERCEPT_FUNCTION(HeapFree); + INTERCEPT_FUNCTION(HeapReAlloc); + INTERCEPT_FUNCTION(HeapAlloc); + + // Undocumented functions must be intercepted by name, not by symbol. + __interception::OverrideFunction("RtlSizeHeap", (uptr)WRAP(RtlSizeHeap), + (uptr *)&REAL(RtlSizeHeap)); + __interception::OverrideFunction("RtlFreeHeap", (uptr)WRAP(RtlFreeHeap), + (uptr *)&REAL(RtlFreeHeap)); + __interception::OverrideFunction("RtlReAllocateHeap", + (uptr)WRAP(RtlReAllocateHeap), + (uptr *)&REAL(RtlReAllocateHeap)); + __interception::OverrideFunction("RtlAllocateHeap", + (uptr)WRAP(RtlAllocateHeap), + (uptr *)&REAL(RtlAllocateHeap)); + } else { +#define INTERCEPT_UCRT_FUNCTION(func) \ + if (!INTERCEPT_FUNCTION_DLLIMPORT("ucrtbase.dll", \ + "api-ms-win-core-heap-l1-1-0.dll", func)) \ + VPrintf(2, "Failed to intercept ucrtbase.dll import %s\n", #func); + INTERCEPT_UCRT_FUNCTION(HeapAlloc); + INTERCEPT_UCRT_FUNCTION(HeapFree); + INTERCEPT_UCRT_FUNCTION(HeapReAlloc); + INTERCEPT_UCRT_FUNCTION(HeapSize); +#undef INTERCEPT_UCRT_FUNCTION + } // Recent versions of ucrtbase.dll appear to be built with PGO and LTCG, which // enable cross-module inlining. This means our _malloc_base hook won't catch // all CRT allocations. This code here patches the import table of @@ -251,16 +545,8 @@ void ReplaceSystemMalloc() { // allocation API will be directed to ASan's heap. We don't currently // intercept all calls to HeapAlloc. If we did, we would have to check on // HeapFree whether the pointer came from ASan of from the system. -#define INTERCEPT_UCRT_FUNCTION(func) \ - if (!INTERCEPT_FUNCTION_DLLIMPORT("ucrtbase.dll", \ - "api-ms-win-core-heap-l1-1-0.dll", func)) \ - VPrintf(2, "Failed to intercept ucrtbase.dll import %s\n", #func); - INTERCEPT_UCRT_FUNCTION(HeapAlloc); - INTERCEPT_UCRT_FUNCTION(HeapFree); - INTERCEPT_UCRT_FUNCTION(HeapReAlloc); - INTERCEPT_UCRT_FUNCTION(HeapSize); -#undef INTERCEPT_UCRT_FUNCTION -#endif + +#endif // defined(ASAN_DYNAMIC) } } // namespace __asan diff --git a/lib/asan/asan_mapping.h b/lib/asan/asan_mapping.h index f3696da62b30..41fb49ee46d4 100644 --- a/lib/asan/asan_mapping.h +++ b/lib/asan/asan_mapping.h @@ -1,9 +1,8 @@ //===-- asan_mapping.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -101,6 +100,13 @@ // || `[0x10000000000000, 0x11ffffffffffff]` || LowShadow || // || `[0x00000000000000, 0x0fffffffffffff]` || LowMem || // +// Default Linux/SPARC64 (52-bit VMA) mapping: +// || `[0x8000000000000, 0xfffffffffffff]` || HighMem || +// || `[0x1080000000000, 0x207ffffffffff]` || HighShadow || +// || `[0x0090000000000, 0x107ffffffffff]` || ShadowGap || +// || `[0x0080000000000, 0x008ffffffffff]` || LowShadow || +// || `[0x0000000000000, 0x007ffffffffff]` || LowMem || +// // Shadow mapping on FreeBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: // || `[0x500000000000, 0x7fffffffffff]` || HighMem || // || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || @@ -154,15 +160,12 @@ static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000 static const u64 kDefaultShadowOffset64 = 1ULL << 44; static const u64 kDefaultShort64bitShadowOffset = 0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G. -static const u64 kIosShadowOffset32 = 1ULL << 30; // 0x40000000 -static const u64 kIosShadowOffset64 = 0x120200000; -static const u64 kIosSimShadowOffset32 = 1ULL << 30; -static const u64 kIosSimShadowOffset64 = kDefaultShadowOffset64; static const u64 kAArch64_ShadowOffset64 = 1ULL << 36; static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000; static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37; static const u64 kPPC64_ShadowOffset64 = 1ULL << 44; static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52; +static const u64 kSPARC64_ShadowOffset64 = 1ULL << 43; // 0x80000000000 static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 @@ -194,11 +197,7 @@ static const u64 kMyriadCacheBitMask32 = 0x40000000ULL; # elif SANITIZER_WINDOWS # define SHADOW_OFFSET kWindowsShadowOffset32 # elif SANITIZER_IOS -# if SANITIZER_IOSSIM -# define SHADOW_OFFSET kIosSimShadowOffset32 -# else -# define SHADOW_OFFSET kIosShadowOffset32 -# endif +# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address # elif SANITIZER_MYRIAD2 # define SHADOW_OFFSET kMyriadShadowOffset32 # else @@ -206,11 +205,7 @@ static const u64 kMyriadCacheBitMask32 = 0x40000000ULL; # endif #else # if SANITIZER_IOS -# if SANITIZER_IOSSIM -# define SHADOW_OFFSET kIosSimShadowOffset64 -# else -# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address -# endif +# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address # elif defined(__aarch64__) # define SHADOW_OFFSET kAArch64_ShadowOffset64 # elif defined(__powerpc64__) @@ -225,6 +220,8 @@ static const u64 kMyriadCacheBitMask32 = 0x40000000ULL; # define SHADOW_OFFSET kDefaultShadowOffset64 # elif defined(__mips64) # define SHADOW_OFFSET kMIPS64_ShadowOffset64 +#elif defined(__sparc__) +#define SHADOW_OFFSET kSPARC64_ShadowOffset64 # elif SANITIZER_WINDOWS64 # define SHADOW_OFFSET __asan_shadow_memory_dynamic_address # else @@ -271,6 +268,8 @@ extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init. #if SANITIZER_MYRIAD2 #include "asan_mapping_myriad.h" +#elif defined(__sparc__) && SANITIZER_WORDSIZE == 64 +#include "asan_mapping_sparc64.h" #else #define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET)) diff --git a/lib/asan/asan_mapping_myriad.h b/lib/asan/asan_mapping_myriad.h index baa3247bd190..6969e3a49310 100644 --- a/lib/asan/asan_mapping_myriad.h +++ b/lib/asan/asan_mapping_myriad.h @@ -1,9 +1,8 @@ //===-- asan_mapping_myriad.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_mapping_sparc64.h b/lib/asan/asan_mapping_sparc64.h new file mode 100644 index 000000000000..432a1816f797 --- /dev/null +++ b/lib/asan/asan_mapping_sparc64.h @@ -0,0 +1,101 @@ +//===-- asan_mapping_sparc64.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// SPARC64-specific definitions for ASan memory mapping. +//===----------------------------------------------------------------------===// +#ifndef ASAN_MAPPING_SPARC64_H +#define ASAN_MAPPING_SPARC64_H + +// This is tailored to the 52-bit VM layout on SPARC-T4 and later. +// The VM space is split into two 51-bit halves at both ends: the low part +// has all the bits above the 51st cleared, while the high part has them set. +// 0xfff8000000000000 - 0xffffffffffffffff +// 0x0000000000000000 - 0x0007ffffffffffff + +#define VMA_BITS 52 +#define HIGH_BITS (64 - VMA_BITS) + +// The idea is to chop the high bits before doing the scaling, so the two +// parts become contiguous again and the usual scheme can be applied. + +#define MEM_TO_SHADOW(mem) \ + ((((mem) << HIGH_BITS) >> (HIGH_BITS + (SHADOW_SCALE))) + (SHADOW_OFFSET)) + +#define kLowMemBeg 0 +#define kLowMemEnd (SHADOW_OFFSET - 1) + +#define kLowShadowBeg SHADOW_OFFSET +#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd) + +// But of course there is the huge hole between the high shadow memory, +// which is in the low part, and the beginning of the high part. + +#define kHighMemBeg (-(1ULL << (VMA_BITS - 1))) + +#define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg) +#define kHighShadowEnd MEM_TO_SHADOW(kHighMemEnd) + +#define kMidShadowBeg 0 +#define kMidShadowEnd 0 + +// With the zero shadow base we can not actually map pages starting from 0. +// This constant is somewhat arbitrary. +#define kZeroBaseShadowStart 0 +#define kZeroBaseMaxShadowStart (1 << 18) + +#define kShadowGapBeg (kLowShadowEnd + 1) +#define kShadowGapEnd (kHighShadowBeg - 1) + +#define kShadowGap2Beg 0 +#define kShadowGap2End 0 + +#define kShadowGap3Beg 0 +#define kShadowGap3End 0 + +namespace __asan { + +static inline bool AddrIsInLowMem(uptr a) { + PROFILE_ASAN_MAPPING(); + return a <= kLowMemEnd; +} + +static inline bool AddrIsInLowShadow(uptr a) { + PROFILE_ASAN_MAPPING(); + return a >= kLowShadowBeg && a <= kLowShadowEnd; +} + +static inline bool AddrIsInMidMem(uptr a) { + PROFILE_ASAN_MAPPING(); + return false; +} + +static inline bool AddrIsInMidShadow(uptr a) { + PROFILE_ASAN_MAPPING(); + return false; +} + +static inline bool AddrIsInHighMem(uptr a) { + PROFILE_ASAN_MAPPING(); + return kHighMemBeg && a >= kHighMemBeg && a <= kHighMemEnd; +} + +static inline bool AddrIsInHighShadow(uptr a) { + PROFILE_ASAN_MAPPING(); + return kHighMemBeg && a >= kHighShadowBeg && a <= kHighShadowEnd; +} + +static inline bool AddrIsInShadowGap(uptr a) { + PROFILE_ASAN_MAPPING(); + return a >= kShadowGapBeg && a <= kShadowGapEnd; +} + +} // namespace __asan + +#endif // ASAN_MAPPING_SPARC64_H diff --git a/lib/asan/asan_memory_profile.cc b/lib/asan/asan_memory_profile.cc index 8c86d9fe49f2..87d874d2f274 100644 --- a/lib/asan/asan_memory_profile.cc +++ b/lib/asan/asan_memory_profile.cc @@ -1,9 +1,8 @@ //===-- asan_memory_profile.cc.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_new_delete.cc b/lib/asan/asan_new_delete.cc index e6053c1fe87f..5f51d12b1b5a 100644 --- a/lib/asan/asan_new_delete.cc +++ b/lib/asan/asan_new_delete.cc @@ -1,9 +1,8 @@ //===-- asan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -72,25 +71,19 @@ enum class align_val_t: size_t {}; // TODO(alekseyshl): throw std::bad_alloc instead of dying on OOM. // For local pool allocation, align to SHADOW_GRANULARITY to match asan // allocator behavior. -#define OPERATOR_NEW_BODY(type, nothrow) \ - if (ALLOCATE_FROM_LOCAL_POOL) {\ - void *res = MemalignFromLocalPool(SHADOW_GRANULARITY, size);\ - if (!nothrow) CHECK(res);\ - return res;\ - }\ - GET_STACK_TRACE_MALLOC;\ - void *res = asan_memalign(0, size, &stack, type);\ - if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\ +#define OPERATOR_NEW_BODY(type, nothrow) \ + MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \ + GET_STACK_TRACE_MALLOC; \ + void *res = asan_memalign(0, size, &stack, type); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ return res; -#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \ - if (ALLOCATE_FROM_LOCAL_POOL) {\ - void *res = MemalignFromLocalPool((uptr)align, size);\ - if (!nothrow) CHECK(res);\ - return res;\ - }\ - GET_STACK_TRACE_MALLOC;\ - void *res = asan_memalign((uptr)align, size, &stack, type);\ - if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\ +#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \ + MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \ + GET_STACK_TRACE_MALLOC; \ + void *res = asan_memalign((uptr)align, size, &stack, type); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ return res; // On OS X it's not enough to just provide our own 'operator new' and diff --git a/lib/asan/asan_poisoning.cc b/lib/asan/asan_poisoning.cc index 1e9c37a13a16..44b87c76e9cc 100644 --- a/lib/asan/asan_poisoning.cc +++ b/lib/asan/asan_poisoning.cc @@ -1,9 +1,8 @@ //===-- asan_poisoning.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_poisoning.h b/lib/asan/asan_poisoning.h index c94794cead89..62dd9bd0edd3 100644 --- a/lib/asan/asan_poisoning.h +++ b/lib/asan/asan_poisoning.h @@ -1,9 +1,8 @@ //===-- asan_poisoning.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #include "asan_internal.h" #include "asan_mapping.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_platform.h" namespace __asan { @@ -39,6 +39,10 @@ void PoisonShadowPartialRightRedzone(uptr addr, ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, u8 value) { DCHECK(!value || CanPoisonMemory()); +#if SANITIZER_FUCHSIA + __sanitizer_fill_shadow(aligned_beg, aligned_size, value, + common_flags()->clear_shadow_mmap_threshold); +#else uptr shadow_beg = MEM_TO_SHADOW(aligned_beg); uptr shadow_end = MEM_TO_SHADOW( aligned_beg + aligned_size - SHADOW_GRANULARITY) + 1; @@ -47,10 +51,6 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, // probably provide higher-level interface for these operations. // For now, just memset on Windows. if (value || SANITIZER_WINDOWS == 1 || - // TODO(mcgrathr): Fuchsia doesn't allow the shadow mapping to be - // changed at all. It doesn't currently have an efficient means - // to zero a bunch of pages, but maybe we should add one. - SANITIZER_FUCHSIA == 1 || // RTEMS doesn't have have pages, let alone a fast way to zero // them, so default to memset. SANITIZER_RTEMS == 1 || @@ -73,6 +73,7 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, ReserveShadowMemoryRange(page_beg, page_end - 1, nullptr); } } +#endif // SANITIZER_FUCHSIA } ALWAYS_INLINE void FastPoisonShadowPartialRightRedzone( diff --git a/lib/asan/asan_posix.cc b/lib/asan/asan_posix.cc index ca99c04b3ad8..5c5e0359ad6c 100644 --- a/lib/asan/asan_posix.cc +++ b/lib/asan/asan_posix.cc @@ -1,9 +1,8 @@ //===-- asan_posix.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_preinit.cc b/lib/asan/asan_preinit.cc index a3986d24f9c3..444998c44176 100644 --- a/lib/asan/asan_preinit.cc +++ b/lib/asan/asan_preinit.cc @@ -1,9 +1,8 @@ //===-- asan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_premap_shadow.cc b/lib/asan/asan_premap_shadow.cc index 229eba99fe0e..6e547718c68e 100644 --- a/lib/asan/asan_premap_shadow.cc +++ b/lib/asan/asan_premap_shadow.cc @@ -1,9 +1,8 @@ //===-- asan_premap_shadow.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_premap_shadow.h b/lib/asan/asan_premap_shadow.h index 41acbdbbb69f..c9c886e8dca7 100644 --- a/lib/asan/asan_premap_shadow.h +++ b/lib/asan/asan_premap_shadow.h @@ -1,9 +1,8 @@ //===-- asan_mapping.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_report.cc b/lib/asan/asan_report.cc index 439712350e46..49067437d9d4 100644 --- a/lib/asan/asan_report.cc +++ b/lib/asan/asan_report.cc @@ -1,9 +1,8 @@ //===-- asan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -191,7 +190,7 @@ class ScopedInErrorReport { void ReportError(const ErrorDescription &description) { // Can only report one error per ScopedInErrorReport. CHECK_EQ(current_error_.kind, kErrorKindInvalid); - current_error_ = description; + internal_memcpy(¤t_error_, &description, sizeof(current_error_)); } static ErrorDescription &CurrentError() { @@ -264,6 +263,13 @@ void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack) { in_report.ReportError(error); } +void ReportReallocArrayOverflow(uptr count, uptr size, + BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorReallocArrayOverflow error(GetCurrentTidOrInvalid(), stack, count, size); + in_report.ReportError(error); +} + void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack) { ScopedInErrorReport in_report(/*fatal*/ true); ErrorPvallocOverflow error(GetCurrentTidOrInvalid(), stack, size); diff --git a/lib/asan/asan_report.h b/lib/asan/asan_report.h index b0c167dda756..dcf60894ef34 100644 --- a/lib/asan/asan_report.h +++ b/lib/asan/asan_report.h @@ -1,9 +1,8 @@ //===-- asan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,6 +61,8 @@ void ReportMallocUsableSizeNotOwned(uptr addr, BufferedStackTrace *stack); void ReportSanitizerGetAllocatedSizeNotOwned(uptr addr, BufferedStackTrace *stack); void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack); +void ReportReallocArrayOverflow(uptr count, uptr size, + BufferedStackTrace *stack); void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack); void ReportInvalidAllocationAlignment(uptr alignment, BufferedStackTrace *stack); diff --git a/lib/asan/asan_rtems.cc b/lib/asan/asan_rtems.cc index b48cc6a75d78..4878f4d67c86 100644 --- a/lib/asan/asan_rtems.cc +++ b/lib/asan/asan_rtems.cc @@ -1,9 +1,8 @@ //===-- asan_rtems.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -184,8 +183,8 @@ static void ThreadStartHook(void *hook, uptr os_id) { // Determine whether we are starting or restarting the thread. if (status == ThreadStatusCreated) // In lieu of AsanThread::ThreadStart. - asanThreadRegistry().StartThread(thread->tid(), os_id, - /*workerthread*/ false, nullptr); + asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular, + nullptr); else { // In a thread restart, a thread may resume execution at an // arbitrary function entry point, with its stack and TLS state diff --git a/lib/asan/asan_rtl.cc b/lib/asan/asan_rtl.cc index 13344f3b8887..db8dcd0689a5 100644 --- a/lib/asan/asan_rtl.cc +++ b/lib/asan/asan_rtl.cc @@ -1,9 +1,8 @@ //===-- asan_rtl.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -598,6 +597,19 @@ void NOINLINE __asan_handle_no_return() { curr_thread->fake_stack()->HandleNoReturn(); } +extern "C" void *__asan_extra_spill_area() { + AsanThread *t = GetCurrentThread(); + CHECK(t); + return t->extra_spill_area(); +} + +void __asan_handle_vfork(void *sp) { + AsanThread *t = GetCurrentThread(); + CHECK(t); + uptr bottom = t->stack_bottom(); + PoisonShadow(bottom, (uptr)sp - bottom, 0); +} + void NOINLINE __asan_set_death_callback(void (*callback)(void)) { SetUserDieCallback(callback); } diff --git a/lib/asan/asan_scariness_score.h b/lib/asan/asan_scariness_score.h index 7f095dd29f29..9e7ba47d82dc 100644 --- a/lib/asan/asan_scariness_score.h +++ b/lib/asan/asan_scariness_score.h @@ -1,9 +1,8 @@ //===-- asan_scariness_score.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_shadow_setup.cc b/lib/asan/asan_shadow_setup.cc index 083926e70aa2..9cfa4e2bd65d 100644 --- a/lib/asan/asan_shadow_setup.cc +++ b/lib/asan/asan_shadow_setup.cc @@ -1,9 +1,8 @@ //===-- asan_shadow_setup.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_stack.cc b/lib/asan/asan_stack.cc index cf7a587fa65a..b244da4fa0ad 100644 --- a/lib/asan/asan_stack.cc +++ b/lib/asan/asan_stack.cc @@ -1,9 +1,8 @@ //===-- asan_stack.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,8 +26,57 @@ u32 GetMallocContextSize() { return atomic_load(&malloc_context_size, memory_order_acquire); } +namespace { + +// ScopedUnwinding is a scope for stacktracing member of a context +class ScopedUnwinding { + public: + explicit ScopedUnwinding(AsanThread *t) : thread(t) { + if (thread) { + can_unwind = !thread->isUnwinding(); + thread->setUnwinding(true); + } + } + ~ScopedUnwinding() { + if (thread) + thread->setUnwinding(false); + } + + bool CanUnwind() const { return can_unwind; } + + private: + AsanThread *thread = nullptr; + bool can_unwind = true; +}; + +} // namespace + } // namespace __asan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __asan; + size = 0; + if (UNLIKELY(!asan_inited)) + return; + request_fast = StackTrace::WillUseFastUnwind(request_fast); + AsanThread *t = GetCurrentThread(); + ScopedUnwinding unwind_scope(t); + if (!unwind_scope.CanUnwind()) + return; + if (request_fast) { + if (t) { + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), + true); + } + return; + } + if (SANITIZER_MIPS && t && + !IsValidFrame(bp, t->stack_top(), t->stack_bottom())) + return; + Unwind(max_depth, pc, bp, context, 0, 0, false); +} + // ------------------ Interface -------------- {{{1 extern "C" { diff --git a/lib/asan/asan_stack.h b/lib/asan/asan_stack.h index 8e9df888d798..3a4b3cefc5de 100644 --- a/lib/asan/asan_stack.h +++ b/lib/asan/asan_stack.h @@ -1,9 +1,8 @@ //===-- asan_stack.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,34 +26,6 @@ static const u32 kDefaultMallocContextSize = 30; void SetMallocContextSize(u32 size); u32 GetMallocContextSize(); -// Get the stack trace with the given pc and bp. -// The pc will be in the position 0 of the resulting stack trace. -// The bp may refer to the current frame or to the caller's frame. -ALWAYS_INLINE -void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, - void *context, bool fast) { -#if SANITIZER_WINDOWS - stack->Unwind(max_depth, pc, bp, context, 0, 0, fast); -#else - AsanThread *t; - stack->size = 0; - if (LIKELY(asan_inited)) { - if ((t = GetCurrentThread()) && !t->isUnwinding()) { - uptr stack_top = t->stack_top(); - uptr stack_bottom = t->stack_bottom(); - ScopedUnwinding unwind_scope(t); - if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { - stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, - fast); - } - } else if (!t && !fast) { - /* If GetCurrentThread() has failed, try to do slow unwind anyways. */ - stack->Unwind(max_depth, pc, bp, context, 0, 0, false); - } - } -#endif // SANITIZER_WINDOWS -} - } // namespace __asan // NOTE: A Rule of thumb is to retrieve stack trace in the interceptors @@ -71,19 +42,19 @@ void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \ } \ } else { \ - GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), 0, fast); \ + stack.Unwind(StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), nullptr, fast, max_size); \ } #define GET_STACK_TRACE_FATAL(pc, bp) \ BufferedStackTrace stack; \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, 0, \ - common_flags()->fast_unwind_on_fatal) + stack.Unwind(pc, bp, nullptr, \ + common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_SIGNAL(sig) \ BufferedStackTrace stack; \ - GetStackTrace(&stack, kStackTraceMax, (sig).pc, (sig).bp, (sig).context, \ - common_flags()->fast_unwind_on_fatal) + stack.Unwind((sig).pc, (sig).bp, (sig).context, \ + common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_FATAL_HERE \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) diff --git a/lib/asan/asan_stats.cc b/lib/asan/asan_stats.cc index b8c68c32dfbf..2f996ce63ccc 100644 --- a/lib/asan/asan_stats.cc +++ b/lib/asan/asan_stats.cc @@ -1,9 +1,8 @@ //===-- asan_stats.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_stats.h b/lib/asan/asan_stats.h index 4605135e166f..d6da6534081f 100644 --- a/lib/asan/asan_stats.h +++ b/lib/asan/asan_stats.h @@ -1,9 +1,8 @@ //===-- asan_stats.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_suppressions.cc b/lib/asan/asan_suppressions.cc index ac8aa023f6ba..118853e61b79 100644 --- a/lib/asan/asan_suppressions.cc +++ b/lib/asan/asan_suppressions.cc @@ -1,9 +1,8 @@ //===-- asan_suppressions.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_suppressions.h b/lib/asan/asan_suppressions.h index 5246b4b30334..9bf297602cfa 100644 --- a/lib/asan/asan_suppressions.h +++ b/lib/asan/asan_suppressions.h @@ -1,9 +1,8 @@ //===-- asan_suppressions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_thread.cc b/lib/asan/asan_thread.cc index 0895e4ce0d93..e63561c2243f 100644 --- a/lib/asan/asan_thread.cc +++ b/lib/asan/asan_thread.cc @@ -1,9 +1,8 @@ //===-- asan_thread.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -246,8 +245,7 @@ void AsanThread::Init(const InitOptions *options) { thread_return_t AsanThread::ThreadStart( tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) { Init(); - asanThreadRegistry().StartThread(tid(), os_id, /*workerthread*/ false, - nullptr); + asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr); if (signal_thread_is_registered) atomic_store(signal_thread_is_registered, 1, memory_order_release); diff --git a/lib/asan/asan_thread.h b/lib/asan/asan_thread.h index 66091921101f..d725e88864eb 100644 --- a/lib/asan/asan_thread.h +++ b/lib/asan/asan_thread.h @@ -1,9 +1,8 @@ //===-- asan_thread.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -131,6 +130,8 @@ class AsanThread { AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } AsanStats &stats() { return stats_; } + void *extra_spill_area() { return &extra_spill_area_; } + private: // NOTE: There is no AsanThread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. @@ -166,18 +167,7 @@ class AsanThread { AsanThreadLocalMallocStorage malloc_storage_; AsanStats stats_; bool unwinding_; -}; - -// ScopedUnwinding is a scope for stacktracing member of a context -class ScopedUnwinding { - public: - explicit ScopedUnwinding(AsanThread *t) : thread(t) { - t->setUnwinding(true); - } - ~ScopedUnwinding() { thread->setUnwinding(false); } - - private: - AsanThread *thread; + uptr extra_spill_area_; }; // Returns a single instance of registry. diff --git a/lib/asan/asan_win.cc b/lib/asan/asan_win.cc index 068f4a5d247d..f7601f3301ea 100644 --- a/lib/asan/asan_win.cc +++ b/lib/asan/asan_win.cc @@ -1,9 +1,8 @@ //===-- asan_win.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,10 +20,10 @@ #include "asan_interceptors.h" #include "asan_internal.h" +#include "asan_mapping.h" #include "asan_report.h" #include "asan_stack.h" #include "asan_thread.h" -#include "asan_mapping.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_mutex.h" #include "sanitizer_common/sanitizer_win.h" @@ -78,7 +77,7 @@ static long WINAPI SEHHandler(EXCEPTION_POINTERS *info) { } INTERCEPTOR_WINAPI(LPTOP_LEVEL_EXCEPTION_FILTER, SetUnhandledExceptionFilter, - LPTOP_LEVEL_EXCEPTION_FILTER ExceptionFilter) { + LPTOP_LEVEL_EXCEPTION_FILTER ExceptionFilter) { CHECK(REAL(SetUnhandledExceptionFilter)); if (ExceptionFilter == &SEHHandler) return REAL(SetUnhandledExceptionFilter)(ExceptionFilter); @@ -105,7 +104,9 @@ INTERCEPTOR_WINAPI(void, RaiseException, void *a, void *b, void *c, void *d) { #ifdef _WIN64 -INTERCEPTOR_WINAPI(int, __C_specific_handler, void *a, void *b, void *c, void *d) { // NOLINT +INTERCEPTOR_WINAPI(EXCEPTION_DISPOSITION, __C_specific_handler, + _EXCEPTION_RECORD *a, void *b, _CONTEXT *c, + _DISPATCHER_CONTEXT *d) { // NOLINT CHECK(REAL(__C_specific_handler)); __asan_handle_no_return(); return REAL(__C_specific_handler)(a, b, c, d); @@ -131,15 +132,14 @@ INTERCEPTOR(int, _except_handler4, void *a, void *b, void *c, void *d) { #endif static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { - AsanThread *t = (AsanThread*)arg; + AsanThread *t = (AsanThread *)arg; SetCurrentThread(t); return t->ThreadStart(GetTid(), /* signal_thread_is_registered */ nullptr); } -INTERCEPTOR_WINAPI(DWORD, CreateThread, - void* security, uptr stack_size, - DWORD (__stdcall *start_routine)(void*), void* arg, - DWORD thr_flags, void* tid) { +INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security, + SIZE_T stack_size, LPTHREAD_START_ROUTINE start_routine, + void *arg, DWORD thr_flags, DWORD *tid) { // Strict init-order checking is thread-hostile. if (flags()->strict_init_order) StopInitOrderChecking(); @@ -149,9 +149,9 @@ INTERCEPTOR_WINAPI(DWORD, CreateThread, bool detached = false; // FIXME: how can we determine it on Windows? u32 current_tid = GetCurrentTidOrInvalid(); AsanThread *t = - AsanThread::Create(start_routine, arg, current_tid, &stack, detached); - return REAL(CreateThread)(security, stack_size, - asan_thread_start, t, thr_flags, tid); + AsanThread::Create(start_routine, arg, current_tid, &stack, detached); + return REAL(CreateThread)(security, stack_size, asan_thread_start, t, + thr_flags, tid); } // }}} @@ -162,10 +162,9 @@ void InitializePlatformInterceptors() { // The interceptors were not designed to be removable, so we have to keep this // module alive for the life of the process. HMODULE pinned; - CHECK(GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_PIN, - (LPCWSTR)&InitializePlatformInterceptors, - &pinned)); + CHECK(GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_PIN, + (LPCWSTR)&InitializePlatformInterceptors, &pinned)); ASAN_INTERCEPT_FUNC(CreateThread); ASAN_INTERCEPT_FUNC(SetUnhandledExceptionFilter); @@ -197,6 +196,30 @@ static bool tsd_key_inited = false; static __declspec(thread) void *fake_tsd = 0; +// https://docs.microsoft.com/en-us/windows/desktop/api/winternl/ns-winternl-_teb +// "[This structure may be altered in future versions of Windows. Applications +// should use the alternate functions listed in this topic.]" +typedef struct _TEB { + PVOID Reserved1[12]; + // PVOID ThreadLocalStoragePointer; is here, at the last field in Reserved1. + PVOID ProcessEnvironmentBlock; + PVOID Reserved2[399]; + BYTE Reserved3[1952]; + PVOID TlsSlots[64]; + BYTE Reserved4[8]; + PVOID Reserved5[26]; + PVOID ReservedForOle; + PVOID Reserved6[4]; + PVOID TlsExpansionSlots; +} TEB, *PTEB; + +constexpr size_t TEB_RESERVED_FIELDS_THREAD_LOCAL_STORAGE_OFFSET = 11; +BOOL IsTlsInitialized() { + PTEB teb = (PTEB)NtCurrentTeb(); + return teb->Reserved1[TEB_RESERVED_FIELDS_THREAD_LOCAL_STORAGE_OFFSET] != + nullptr; +} + void AsanTSDInit(void (*destructor)(void *tsd)) { // FIXME: we're ignoring the destructor for now. tsd_key_inited = true; @@ -204,7 +227,7 @@ void AsanTSDInit(void (*destructor)(void *tsd)) { void *AsanTSDGet() { CHECK(tsd_key_inited); - return fake_tsd; + return IsTlsInitialized() ? fake_tsd : nullptr; } void AsanTSDSet(void *tsd) { @@ -212,9 +235,7 @@ void AsanTSDSet(void *tsd) { fake_tsd = tsd; } -void PlatformTSDDtor(void *tsd) { - AsanThread::TSDDtor(tsd); -} +void PlatformTSDDtor(void *tsd) { AsanThread::TSDDtor(tsd); } // }}} // ---------------------- Various stuff ---------------- {{{ @@ -245,9 +266,7 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } -void AsanOnDeadlySignal(int, void *siginfo, void *context) { - UNIMPLEMENTED(); -} +void AsanOnDeadlySignal(int, void *siginfo, void *context) { UNIMPLEMENTED(); } #if SANITIZER_WINDOWS64 // Exception handler for dealing with shadow memory. @@ -256,7 +275,9 @@ ShadowExceptionHandler(PEXCEPTION_POINTERS exception_pointers) { uptr page_size = GetPageSizeCached(); // Only handle access violations. if (exception_pointers->ExceptionRecord->ExceptionCode != - EXCEPTION_ACCESS_VIOLATION) { + EXCEPTION_ACCESS_VIOLATION || + exception_pointers->ExceptionRecord->NumberParameters < 2) { + __asan_handle_no_return(); return EXCEPTION_CONTINUE_SEARCH; } @@ -265,7 +286,10 @@ ShadowExceptionHandler(PEXCEPTION_POINTERS exception_pointers) { (uptr)(exception_pointers->ExceptionRecord->ExceptionInformation[1]); // Check valid shadow range. - if (!AddrIsInShadow(addr)) return EXCEPTION_CONTINUE_SEARCH; + if (!AddrIsInShadow(addr)) { + __asan_handle_no_return(); + return EXCEPTION_CONTINUE_SEARCH; + } // This is an access violation while trying to read from the shadow. Commit // the relevant page and let execution continue. @@ -276,7 +300,8 @@ ShadowExceptionHandler(PEXCEPTION_POINTERS exception_pointers) { // Commit the page. uptr result = (uptr)::VirtualAlloc((LPVOID)page, page_size, MEM_COMMIT, PAGE_READWRITE); - if (result != page) return EXCEPTION_CONTINUE_SEARCH; + if (result != page) + return EXCEPTION_CONTINUE_SEARCH; // The page mapping succeeded, so continue execution as usual. return EXCEPTION_CONTINUE_EXECUTION; @@ -293,7 +318,7 @@ void InitializePlatformExceptionHandlers() { } bool IsSystemHeapAddress(uptr addr) { - return ::HeapValidate(GetProcessHeap(), 0, (void*)addr) != FALSE; + return ::HeapValidate(GetProcessHeap(), 0, (void *)addr) != FALSE; } // We want to install our own exception handler (EH) to print helpful reports @@ -312,8 +337,7 @@ bool IsSystemHeapAddress(uptr addr) { // asan_dynamic_runtime_thunk.lib to all the modules, thus __asan_set_seh_filter // will be called for each instrumented module. This ensures that at least one // __asan_set_seh_filter call happens after the .exe module CRT is initialized. -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int __asan_set_seh_filter() { +extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __asan_set_seh_filter() { // We should only store the previous handler if it's not our own handler in // order to avoid loops in the EH chain. auto prev_seh_handler = SetUnhandledExceptionFilter(SEHHandler); @@ -347,14 +371,28 @@ __declspec(allocate(".CRT$XCAB")) int (*__intercept_seh)() = // which run before the CRT. Users also add code to .CRT$XLC, so it's important // to run our initializers first. static void NTAPI asan_thread_init(void *module, DWORD reason, void *reserved) { - if (reason == DLL_PROCESS_ATTACH) __asan_init(); + if (reason == DLL_PROCESS_ATTACH) + __asan_init(); } #pragma section(".CRT$XLAB", long, read) // NOLINT -__declspec(allocate(".CRT$XLAB")) void (NTAPI *__asan_tls_init)(void *, - unsigned long, void *) = asan_thread_init; +__declspec(allocate(".CRT$XLAB")) void(NTAPI *__asan_tls_init)( + void *, unsigned long, void *) = asan_thread_init; #endif +static void NTAPI asan_thread_exit(void *module, DWORD reason, void *reserved) { + if (reason == DLL_THREAD_DETACH) { + // Unpoison the thread's stack because the memory may be re-used. + NT_TIB *tib = (NT_TIB *)NtCurrentTeb(); + uptr stackSize = (uptr)tib->StackBase - (uptr)tib->StackLimit; + __asan_unpoison_memory_region(tib->StackLimit, stackSize); + } +} + +#pragma section(".CRT$XLY", long, read) // NOLINT +__declspec(allocate(".CRT$XLY")) void(NTAPI *__asan_tls_exit)( + void *, unsigned long, void *) = asan_thread_exit; + WIN_FORCE_LINK(__asan_dso_reg_hook) // }}} diff --git a/lib/asan/asan_win_dll_thunk.cc b/lib/asan/asan_win_dll_thunk.cc index df593ab92de6..47b3948c5aea 100644 --- a/lib/asan/asan_win_dll_thunk.cc +++ b/lib/asan/asan_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- asan_win_dll_thunk.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_win_dynamic_runtime_thunk.cc b/lib/asan/asan_win_dynamic_runtime_thunk.cc index 416c73b23629..cf4a59842c4f 100644 --- a/lib/asan/asan_win_dynamic_runtime_thunk.cc +++ b/lib/asan/asan_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- asan_win_dynamic_runtime_thunk.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/asan/asan_win_weak_interception.cc b/lib/asan/asan_win_weak_interception.cc index ca26f914cf5f..19965ca473b4 100644 --- a/lib/asan/asan_win_weak_interception.cc +++ b/lib/asan/asan_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- asan_win_weak_interception.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in Address Sanitizer when it is implemented as diff --git a/lib/builtins/aarch64/chkstk.S b/lib/builtins/aarch64/chkstk.S index 89ec90b08a13..01f90366f030 100644 --- a/lib/builtins/aarch64/chkstk.S +++ b/lib/builtins/aarch64/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/absvdi2.c b/lib/builtins/absvdi2.c index 682c2355d2ac..b9566cd874fe 100644 --- a/lib/builtins/absvdi2.c +++ b/lib/builtins/absvdi2.c @@ -1,29 +1,25 @@ -/*===-- absvdi2.c - Implement __absvdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * This file implements __absvdi2 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- absvdi2.c - Implement __absvdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __absvdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: absolute value */ +// Returns: absolute value -/* Effects: aborts if abs(x) < 0 */ +// Effects: aborts if abs(x) < 0 -COMPILER_RT_ABI di_int -__absvdi2(di_int a) -{ - const int N = (int)(sizeof(di_int) * CHAR_BIT); - if (a == ((di_int)1 << (N-1))) - compilerrt_abort(); - const di_int t = a >> (N - 1); - return (a ^ t) - t; +COMPILER_RT_ABI di_int __absvdi2(di_int a) { + const int N = (int)(sizeof(di_int) * CHAR_BIT); + if (a == ((di_int)1 << (N - 1))) + compilerrt_abort(); + const di_int t = a >> (N - 1); + return (a ^ t) - t; } diff --git a/lib/builtins/absvsi2.c b/lib/builtins/absvsi2.c index 4812af815982..44ada169e7e6 100644 --- a/lib/builtins/absvsi2.c +++ b/lib/builtins/absvsi2.c @@ -1,29 +1,25 @@ -/* ===-- absvsi2.c - Implement __absvsi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __absvsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- absvsi2.c - Implement __absvsi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __absvsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: absolute value */ +// Returns: absolute value -/* Effects: aborts if abs(x) < 0 */ +// Effects: aborts if abs(x) < 0 -COMPILER_RT_ABI si_int -__absvsi2(si_int a) -{ - const int N = (int)(sizeof(si_int) * CHAR_BIT); - if (a == (1 << (N-1))) - compilerrt_abort(); - const si_int t = a >> (N - 1); - return (a ^ t) - t; +COMPILER_RT_ABI si_int __absvsi2(si_int a) { + const int N = (int)(sizeof(si_int) * CHAR_BIT); + if (a == (1 << (N - 1))) + compilerrt_abort(); + const si_int t = a >> (N - 1); + return (a ^ t) - t; } diff --git a/lib/builtins/absvti2.c b/lib/builtins/absvti2.c index 7927770c9ab3..491d99d7ce0f 100644 --- a/lib/builtins/absvti2.c +++ b/lib/builtins/absvti2.c @@ -1,34 +1,29 @@ -/* ===-- absvti2.c - Implement __absvdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __absvti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- absvti2.c - Implement __absvdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __absvti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: absolute value */ +// Returns: absolute value -/* Effects: aborts if abs(x) < 0 */ +// Effects: aborts if abs(x) < 0 -COMPILER_RT_ABI ti_int -__absvti2(ti_int a) -{ - const int N = (int)(sizeof(ti_int) * CHAR_BIT); - if (a == ((ti_int)1 << (N-1))) - compilerrt_abort(); - const ti_int s = a >> (N - 1); - return (a ^ s) - s; +COMPILER_RT_ABI ti_int __absvti2(ti_int a) { + const int N = (int)(sizeof(ti_int) * CHAR_BIT); + if (a == ((ti_int)1 << (N - 1))) + compilerrt_abort(); + const ti_int s = a >> (N - 1); + return (a ^ s) - s; } -#endif /* CRT_HAS_128BIT */ - +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/adddf3.c b/lib/builtins/adddf3.c index 9a3901312e51..f2727fafcabe 100644 --- a/lib/builtins/adddf3.c +++ b/lib/builtins/adddf3.c @@ -1,9 +1,8 @@ //===-- lib/adddf3.c - Double-precision addition ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define DOUBLE_PRECISION #include "fp_add_impl.inc" -COMPILER_RT_ABI double __adddf3(double a, double b){ - return __addXf3__(a, b); -} +COMPILER_RT_ABI double __adddf3(double a, double b) { return __addXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_dadd(double a, double b) { - return __adddf3(a, b); -} +AEABI_RTABI double __aeabi_dadd(double a, double b) { return __adddf3(a, b); } #else -AEABI_RTABI double __aeabi_dadd(double a, double b) COMPILER_RT_ALIAS(__adddf3); +COMPILER_RT_ALIAS(__adddf3, __aeabi_dadd) #endif #endif diff --git a/lib/builtins/addsf3.c b/lib/builtins/addsf3.c index c5c1a41c3611..8fe8622aadd9 100644 --- a/lib/builtins/addsf3.c +++ b/lib/builtins/addsf3.c @@ -1,9 +1,8 @@ //===-- lib/addsf3.c - Single-precision addition ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define SINGLE_PRECISION #include "fp_add_impl.inc" -COMPILER_RT_ABI float __addsf3(float a, float b) { - return __addXf3__(a, b); -} +COMPILER_RT_ABI float __addsf3(float a, float b) { return __addXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_fadd(float a, float b) { - return __addsf3(a, b); -} +AEABI_RTABI float __aeabi_fadd(float a, float b) { return __addsf3(a, b); } #else -AEABI_RTABI float __aeabi_fadd(float a, float b) COMPILER_RT_ALIAS(__addsf3); +COMPILER_RT_ALIAS(__addsf3, __aeabi_fadd) #endif #endif diff --git a/lib/builtins/addtf3.c b/lib/builtins/addtf3.c index e4bbe0227ae6..570472a14554 100644 --- a/lib/builtins/addtf3.c +++ b/lib/builtins/addtf3.c @@ -1,9 +1,8 @@ //===-- lib/addtf3.c - Quad-precision addition --------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,8 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) #include "fp_add_impl.inc" -COMPILER_RT_ABI long double __addtf3(long double a, long double b){ - return __addXf3__(a, b); +COMPILER_RT_ABI long double __addtf3(long double a, long double b) { + return __addXf3__(a, b); } #endif diff --git a/lib/builtins/addvdi3.c b/lib/builtins/addvdi3.c index 0da38945679d..28661fda8498 100644 --- a/lib/builtins/addvdi3.c +++ b/lib/builtins/addvdi3.c @@ -1,36 +1,29 @@ -/* ===-- addvdi3.c - Implement __addvdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __addvdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- addvdi3.c - Implement __addvdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __addvdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a + b */ +// Returns: a + b -/* Effects: aborts if a + b overflows */ +// Effects: aborts if a + b overflows -COMPILER_RT_ABI di_int -__addvdi3(di_int a, di_int b) -{ - di_int s = (du_int) a + (du_int) b; - if (b >= 0) - { - if (s < a) - compilerrt_abort(); - } - else - { - if (s >= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI di_int __addvdi3(di_int a, di_int b) { + di_int s = (du_int)a + (du_int)b; + if (b >= 0) { + if (s < a) + compilerrt_abort(); + } else { + if (s >= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/addvsi3.c b/lib/builtins/addvsi3.c index 94ca726f42b9..40400237527b 100644 --- a/lib/builtins/addvsi3.c +++ b/lib/builtins/addvsi3.c @@ -1,36 +1,29 @@ -/* ===-- addvsi3.c - Implement __addvsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __addvsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- addvsi3.c - Implement __addvsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __addvsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a + b */ +// Returns: a + b -/* Effects: aborts if a + b overflows */ +// Effects: aborts if a + b overflows -COMPILER_RT_ABI si_int -__addvsi3(si_int a, si_int b) -{ - si_int s = (su_int) a + (su_int) b; - if (b >= 0) - { - if (s < a) - compilerrt_abort(); - } - else - { - if (s >= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI si_int __addvsi3(si_int a, si_int b) { + si_int s = (su_int)a + (su_int)b; + if (b >= 0) { + if (s < a) + compilerrt_abort(); + } else { + if (s >= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/addvti3.c b/lib/builtins/addvti3.c index c224de60aab0..aa709875d243 100644 --- a/lib/builtins/addvti3.c +++ b/lib/builtins/addvti3.c @@ -1,40 +1,33 @@ -/* ===-- addvti3.c - Implement __addvti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __addvti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- addvti3.c - Implement __addvti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __addvti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a + b */ +// Returns: a + b -/* Effects: aborts if a + b overflows */ +// Effects: aborts if a + b overflows -COMPILER_RT_ABI ti_int -__addvti3(ti_int a, ti_int b) -{ - ti_int s = (tu_int) a + (tu_int) b; - if (b >= 0) - { - if (s < a) - compilerrt_abort(); - } - else - { - if (s >= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI ti_int __addvti3(ti_int a, ti_int b) { + ti_int s = (tu_int)a + (tu_int)b; + if (b >= 0) { + if (s < a) + compilerrt_abort(); + } else { + if (s >= a) + compilerrt_abort(); + } + return s; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/apple_versioning.c b/lib/builtins/apple_versioning.c index 3797a1ab02da..f87b42820c15 100644 --- a/lib/builtins/apple_versioning.c +++ b/lib/builtins/apple_versioning.c @@ -1,47 +1,42 @@ -/* ===-- apple_versioning.c - Adds versioning symbols for ld ---------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ - +//===-- apple_versioning.c - Adds versioning symbols for ld ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #if __APPLE__ - #include <Availability.h> - - #if __IPHONE_OS_VERSION_MIN_REQUIRED - #define NOT_HERE_BEFORE_10_6(sym) - #define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ - extern const char sym##_tmp61 __asm("$ld$hide$os6.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp61 = 0; \ - extern const char sym##_tmp60 __asm("$ld$hide$os6.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp60 = 0; \ - extern const char sym##_tmp51 __asm("$ld$hide$os5.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp51 = 0; \ - extern const char sym##_tmp50 __asm("$ld$hide$os5.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp50 = 0; - #else - #define NOT_HERE_BEFORE_10_6(sym) \ - extern const char sym##_tmp4 __asm("$ld$hide$os10.4$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ - extern const char sym##_tmp5 __asm("$ld$hide$os10.5$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp5 = 0; - #define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ - extern const char sym##_tmp8 __asm("$ld$hide$os10.8$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp8 = 0; \ - extern const char sym##_tmp7 __asm("$ld$hide$os10.7$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp7 = 0; \ - extern const char sym##_tmp6 __asm("$ld$hide$os10.6$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp6 = 0; - #endif +#include <Availability.h> +#if __IPHONE_OS_VERSION_MIN_REQUIRED +#define NOT_HERE_BEFORE_10_6(sym) +#define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ + extern const char sym##_tmp61 __asm("$ld$hide$os6.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp61 = 0; \ + extern const char sym##_tmp60 __asm("$ld$hide$os6.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp60 = 0; \ + extern const char sym##_tmp51 __asm("$ld$hide$os5.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp51 = 0; \ + extern const char sym##_tmp50 __asm("$ld$hide$os5.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp50 = 0; +#else +#define NOT_HERE_BEFORE_10_6(sym) \ + extern const char sym##_tmp4 __asm("$ld$hide$os10.4$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ + extern const char sym##_tmp5 __asm("$ld$hide$os10.5$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp5 = 0; +#define NOT_HERE_IN_10_8_AND_EARLIER(sym) \ + extern const char sym##_tmp8 __asm("$ld$hide$os10.8$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp8 = 0; \ + extern const char sym##_tmp7 __asm("$ld$hide$os10.7$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp7 = 0; \ + extern const char sym##_tmp6 __asm("$ld$hide$os10.6$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp6 = 0; +#endif -/* Symbols in libSystem.dylib in 10.6 and later, - * but are in libgcc_s.dylib in earlier versions - */ +// Symbols in libSystem.dylib in 10.6 and later, +// but are in libgcc_s.dylib in earlier versions NOT_HERE_BEFORE_10_6(__absvdi2) NOT_HERE_BEFORE_10_6(__absvsi2) @@ -143,14 +138,13 @@ NOT_HERE_BEFORE_10_6(__udivti3) NOT_HERE_BEFORE_10_6(__umoddi3) NOT_HERE_BEFORE_10_6(__umodti3) - #if __ppc__ NOT_HERE_BEFORE_10_6(__gcc_qadd) NOT_HERE_BEFORE_10_6(__gcc_qdiv) NOT_HERE_BEFORE_10_6(__gcc_qmul) NOT_HERE_BEFORE_10_6(__gcc_qsub) NOT_HERE_BEFORE_10_6(__trampoline_setup) -#endif /* __ppc__ */ +#endif // __ppc__ NOT_HERE_IN_10_8_AND_EARLIER(__atomic_compare_exchange) NOT_HERE_IN_10_8_AND_EARLIER(__atomic_compare_exchange_1) @@ -201,24 +195,23 @@ NOT_HERE_IN_10_8_AND_EARLIER(__atomic_store_2) NOT_HERE_IN_10_8_AND_EARLIER(__atomic_store_4) NOT_HERE_IN_10_8_AND_EARLIER(__atomic_store_8) - #if __arm__ && __DYNAMIC__ - #define NOT_HERE_UNTIL_AFTER_4_3(sym) \ - extern const char sym##_tmp1 __asm("$ld$hide$os3.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp1 = 0; \ - extern const char sym##_tmp2 __asm("$ld$hide$os3.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp2 = 0; \ - extern const char sym##_tmp3 __asm("$ld$hide$os3.2$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp3 = 0; \ - extern const char sym##_tmp4 __asm("$ld$hide$os4.0$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ - extern const char sym##_tmp5 __asm("$ld$hide$os4.1$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp5 = 0; \ - extern const char sym##_tmp6 __asm("$ld$hide$os4.2$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp6 = 0; \ - extern const char sym##_tmp7 __asm("$ld$hide$os4.3$_" #sym ); \ - __attribute__((visibility("default"))) const char sym##_tmp7 = 0; - +#define NOT_HERE_UNTIL_AFTER_4_3(sym) \ + extern const char sym##_tmp1 __asm("$ld$hide$os3.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp1 = 0; \ + extern const char sym##_tmp2 __asm("$ld$hide$os3.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp2 = 0; \ + extern const char sym##_tmp3 __asm("$ld$hide$os3.2$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp3 = 0; \ + extern const char sym##_tmp4 __asm("$ld$hide$os4.0$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ + extern const char sym##_tmp5 __asm("$ld$hide$os4.1$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp5 = 0; \ + extern const char sym##_tmp6 __asm("$ld$hide$os4.2$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp6 = 0; \ + extern const char sym##_tmp7 __asm("$ld$hide$os4.3$_" #sym); \ + __attribute__((visibility("default"))) const char sym##_tmp7 = 0; + NOT_HERE_UNTIL_AFTER_4_3(__absvdi2) NOT_HERE_UNTIL_AFTER_4_3(__absvsi2) NOT_HERE_UNTIL_AFTER_4_3(__adddf3) @@ -339,12 +332,8 @@ NOT_HERE_UNTIL_AFTER_4_3(__divmodsi4) NOT_HERE_UNTIL_AFTER_4_3(__udivmodsi4) #endif // __arm__ && __DYNAMIC__ - - - - -#else /* !__APPLE__ */ +#else // !__APPLE__ extern int avoid_empty_file; -#endif /* !__APPLE__*/ +#endif // !__APPLE__ diff --git a/lib/builtins/arm/adddf3vfp.S b/lib/builtins/arm/adddf3vfp.S index 8e476cad1624..1a271db0847c 100644 --- a/lib/builtins/arm/adddf3vfp.S +++ b/lib/builtins/arm/adddf3vfp.S @@ -1,20 +1,18 @@ //===-- adddf3vfp.S - Implement adddf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // double __adddf3vfp(double a, double b) { return a + b; } // // Adds two double precision floating point numbers using the Darwin // calling convention where double arguments are passsed in GPR pairs -// + .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__adddf3vfp) @@ -23,7 +21,7 @@ DEFINE_COMPILERRT_FUNCTION(__adddf3vfp) #else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vadd.f64 d6, d6, d7 + vadd.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair #endif bx lr diff --git a/lib/builtins/arm/addsf3.S b/lib/builtins/arm/addsf3.S index 74723cbeff74..aa4d40473edb 100644 --- a/lib/builtins/arm/addsf3.S +++ b/lib/builtins/arm/addsf3.S @@ -1,17 +1,16 @@ -/*===-- addsf3.S - Adds two single precision floating pointer numbers-----===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __addsf3 (single precision floating pointer number - * addition with the IEEE-754 default rounding (to nearest, ties to even) - * function for the ARM Thumb1 ISA. - * - *===----------------------------------------------------------------------===*/ +//===-- addsf3.S - Adds two single precision floating pointer numbers-----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __addsf3 (single precision floating pointer number +// addition with the IEEE-754 default rounding (to nearest, ties to even) +// function for the ARM Thumb1 ISA. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" #define significandBits 23 @@ -29,9 +28,9 @@ DEFINE_COMPILERRT_THUMB_FUNCTION(__addsf3) // Get the absolute value of a and b. lsls r2, r0, #1 lsls r3, r1, #1 - lsrs r2, r2, #1 /* aAbs */ + lsrs r2, r2, #1 // aAbs beq LOCAL_LABEL(a_zero_nan_inf) - lsrs r3, r3, #1 /* bAbs */ + lsrs r3, r3, #1 // bAbs beq LOCAL_LABEL(zero_nan_inf) // Detect if a or b is infinity or Nan. @@ -55,9 +54,9 @@ LOCAL_LABEL(no_swap): // Get the significands and shift them to give us round, guard and sticky. lsls r4, r0, #(typeWidth - significandBits) - lsrs r4, r4, #(typeWidth - significandBits - 3) /* aSignificand << 3 */ + lsrs r4, r4, #(typeWidth - significandBits - 3) // aSignificand << 3 lsls r5, r1, #(typeWidth - significandBits) - lsrs r5, r5, #(typeWidth - significandBits - 3) /* bSignificand << 3 */ + lsrs r5, r5, #(typeWidth - significandBits - 3) // bSignificand << 3 // Get the implicitBit. movs r6, #1 @@ -199,7 +198,7 @@ LOCAL_LABEL(do_substraction): beq 1f movs r7, #1 1: - lsrs r4, r6 /* aSignificand >> shift */ + lsrs r4, r6 // aSignificand >> shift orrs r4, r7 b LOCAL_LABEL(form_result) diff --git a/lib/builtins/arm/addsf3vfp.S b/lib/builtins/arm/addsf3vfp.S index 8871efdcc5d1..c9d1fd1509a8 100644 --- a/lib/builtins/arm/addsf3vfp.S +++ b/lib/builtins/arm/addsf3vfp.S @@ -1,9 +1,8 @@ //===-- addsf3vfp.S - Implement addsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_cdcmp.S b/lib/builtins/arm/aeabi_cdcmp.S index adc2d55d90f5..bd039a0329ea 100644 --- a/lib/builtins/arm/aeabi_cdcmp.S +++ b/lib/builtins/arm/aeabi_cdcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_cdcmp.S - EABI cdcmp* implementation ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c b/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c index 7578433a1df7..7bae8743fcf2 100644 --- a/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c +++ b/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c @@ -1,16 +1,15 @@ //===-- lib/arm/aeabi_cdcmpeq_helper.c - Helper for cdcmpeq ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include <stdint.h> #include "../int_lib.h" +#include <stdint.h> -AEABI_RTABI __attribute__((visibility("hidden"))) -int __aeabi_cdcmpeq_check_nan(double a, double b) { - return __builtin_isnan(a) || __builtin_isnan(b); +AEABI_RTABI __attribute__((visibility("hidden"))) int +__aeabi_cdcmpeq_check_nan(double a, double b) { + return __builtin_isnan(a) || __builtin_isnan(b); } diff --git a/lib/builtins/arm/aeabi_cfcmp.S b/lib/builtins/arm/aeabi_cfcmp.S index 4b1de997687f..a26cb2a3ce16 100644 --- a/lib/builtins/arm/aeabi_cfcmp.S +++ b/lib/builtins/arm/aeabi_cfcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_cfcmp.S - EABI cfcmp* implementation ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c b/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c index 43dde9a49597..25407337d075 100644 --- a/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c +++ b/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c @@ -1,16 +1,15 @@ //===-- lib/arm/aeabi_cfcmpeq_helper.c - Helper for cdcmpeq ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include <stdint.h> #include "../int_lib.h" +#include <stdint.h> -AEABI_RTABI __attribute__((visibility("hidden"))) -int __aeabi_cfcmpeq_check_nan(float a, float b) { - return __builtin_isnan(a) || __builtin_isnan(b); +AEABI_RTABI __attribute__((visibility("hidden"))) int +__aeabi_cfcmpeq_check_nan(float a, float b) { + return __builtin_isnan(a) || __builtin_isnan(b); } diff --git a/lib/builtins/arm/aeabi_dcmp.S b/lib/builtins/arm/aeabi_dcmp.S index 9fa78b461248..5f720670ddd7 100644 --- a/lib/builtins/arm/aeabi_dcmp.S +++ b/lib/builtins/arm/aeabi_dcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_dcmp.S - EABI dcmp* implementation ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_div0.c b/lib/builtins/arm/aeabi_div0.c index dc3031326e37..7e8862321d82 100644 --- a/lib/builtins/arm/aeabi_div0.c +++ b/lib/builtins/arm/aeabi_div0.c @@ -1,34 +1,30 @@ -/* ===-- aeabi_div0.c - ARM Runtime ABI support routines for compiler-rt ---=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements the division by zero helper routines as specified by the - * Run-time ABI for the ARM Architecture. - * - * ===----------------------------------------------------------------------=== - */ +//===-- aeabi_div0.c - ARM Runtime ABI support routines for compiler-rt ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the division by zero helper routines as specified by the +// Run-time ABI for the ARM Architecture. +// +//===----------------------------------------------------------------------===// -/* - * RTABI 4.3.2 - Division by zero - * - * The *div0 functions: - * - Return the value passed to them as a parameter - * - Or, return a fixed value defined by the execution environment (such as 0) - * - Or, raise a signal (often SIGFPE) or throw an exception, and do not return - * - * An application may provide its own implementations of the *div0 functions to - * for a particular behaviour from the *div and *divmod functions called out of - * line. - */ +// RTABI 4.3.2 - Division by zero +// +// The *div0 functions: +// - Return the value passed to them as a parameter +// - Or, return a fixed value defined by the execution environment (such as 0) +// - Or, raise a signal (often SIGFPE) or throw an exception, and do not return +// +// An application may provide its own implementations of the *div0 functions to +// for a particular behaviour from the *div and *divmod functions called out of +// line. #include "../int_lib.h" -/* provide an unused declaration to pacify pendantic compilation */ +// provide an unused declaration to pacify pendantic compilation extern unsigned char declaration; #if defined(__ARM_EABI__) @@ -37,9 +33,8 @@ __aeabi_idiv0(int return_value) { return return_value; } -AEABI_RTABI long long __attribute__((weak)) __attribute__((visibility("hidden"))) -__aeabi_ldiv0(long long return_value) { +AEABI_RTABI long long __attribute__((weak)) +__attribute__((visibility("hidden"))) __aeabi_ldiv0(long long return_value) { return return_value; } #endif - diff --git a/lib/builtins/arm/aeabi_drsub.c b/lib/builtins/arm/aeabi_drsub.c index 1254886086fb..e4e8dc051468 100644 --- a/lib/builtins/arm/aeabi_drsub.c +++ b/lib/builtins/arm/aeabi_drsub.c @@ -1,19 +1,14 @@ //===-- lib/arm/aeabi_drsub.c - Double-precision subtraction --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "../fp_lib.h" -AEABI_RTABI fp_t -__aeabi_dsub(fp_t, fp_t); +AEABI_RTABI fp_t __aeabi_dsub(fp_t, fp_t); -AEABI_RTABI fp_t -__aeabi_drsub(fp_t a, fp_t b) { - return __aeabi_dsub(b, a); -} +AEABI_RTABI fp_t __aeabi_drsub(fp_t a, fp_t b) { return __aeabi_dsub(b, a); } diff --git a/lib/builtins/arm/aeabi_fcmp.S b/lib/builtins/arm/aeabi_fcmp.S index ea5b96c21d57..cd311b417096 100644 --- a/lib/builtins/arm/aeabi_fcmp.S +++ b/lib/builtins/arm/aeabi_fcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_fcmp.S - EABI fcmp* implementation ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_frsub.c b/lib/builtins/arm/aeabi_frsub.c index 34f2303745bc..9a363248f14f 100644 --- a/lib/builtins/arm/aeabi_frsub.c +++ b/lib/builtins/arm/aeabi_frsub.c @@ -1,19 +1,14 @@ //===-- lib/arm/aeabi_frsub.c - Single-precision subtraction --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "../fp_lib.h" -AEABI_RTABI fp_t -__aeabi_fsub(fp_t, fp_t); +AEABI_RTABI fp_t __aeabi_fsub(fp_t, fp_t); -AEABI_RTABI fp_t -__aeabi_frsub(fp_t a, fp_t b) { - return __aeabi_fsub(b, a); -} +AEABI_RTABI fp_t __aeabi_frsub(fp_t a, fp_t b) { return __aeabi_fsub(b, a); } diff --git a/lib/builtins/arm/aeabi_idivmod.S b/lib/builtins/arm/aeabi_idivmod.S index 9c9c80ab5a7b..bb80e4b96fc1 100644 --- a/lib/builtins/arm/aeabi_idivmod.S +++ b/lib/builtins/arm/aeabi_idivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_idivmod.S - EABI idivmod implementation ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_ldivmod.S b/lib/builtins/arm/aeabi_ldivmod.S index 038ae5d723a3..d0d06be6f329 100644 --- a/lib/builtins/arm/aeabi_ldivmod.S +++ b/lib/builtins/arm/aeabi_ldivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_ldivmod.S - EABI ldivmod implementation ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memcmp.S b/lib/builtins/arm/aeabi_memcmp.S index e86d6113760e..41637289c1bc 100644 --- a/lib/builtins/arm/aeabi_memcmp.S +++ b/lib/builtins/arm/aeabi_memcmp.S @@ -1,9 +1,8 @@ //===-- aeabi_memcmp.S - EABI memcmp implementation -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memcpy.S b/lib/builtins/arm/aeabi_memcpy.S index e83c5fd4dbb3..93e1b05d54ce 100644 --- a/lib/builtins/arm/aeabi_memcpy.S +++ b/lib/builtins/arm/aeabi_memcpy.S @@ -1,9 +1,8 @@ //===-- aeabi_memcpy.S - EABI memcpy implementation -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memmove.S b/lib/builtins/arm/aeabi_memmove.S index ee28300e46f2..c2f0fa4cd8ff 100644 --- a/lib/builtins/arm/aeabi_memmove.S +++ b/lib/builtins/arm/aeabi_memmove.S @@ -1,9 +1,8 @@ //===-- aeabi_memmove.S - EABI memmove implementation --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_memset.S b/lib/builtins/arm/aeabi_memset.S index 0a678d7627e7..2aa8ec0c4679 100644 --- a/lib/builtins/arm/aeabi_memset.S +++ b/lib/builtins/arm/aeabi_memset.S @@ -1,9 +1,8 @@ //===-- aeabi_memset.S - EABI memset implementation -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_uidivmod.S b/lib/builtins/arm/aeabi_uidivmod.S index 88a4a6d8bc12..df030769fd40 100644 --- a/lib/builtins/arm/aeabi_uidivmod.S +++ b/lib/builtins/arm/aeabi_uidivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_uidivmod.S - EABI uidivmod implementation -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/aeabi_uldivmod.S b/lib/builtins/arm/aeabi_uldivmod.S index be343b6bc826..4fc97704d3c5 100644 --- a/lib/builtins/arm/aeabi_uldivmod.S +++ b/lib/builtins/arm/aeabi_uldivmod.S @@ -1,9 +1,8 @@ //===-- aeabi_uldivmod.S - EABI uldivmod implementation -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/bswapdi2.S b/lib/builtins/arm/bswapdi2.S index e9db8bac7994..271df8bd6ad0 100644 --- a/lib/builtins/arm/bswapdi2.S +++ b/lib/builtins/arm/bswapdi2.S @@ -1,9 +1,8 @@ //===------- bswapdi2 - Implement bswapdi2 --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/bswapsi2.S b/lib/builtins/arm/bswapsi2.S index 1f6eed5c1bbf..07cc3d89190e 100644 --- a/lib/builtins/arm/bswapsi2.S +++ b/lib/builtins/arm/bswapsi2.S @@ -1,9 +1,8 @@ //===------- bswapsi2 - Implement bswapsi2 --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/chkstk.S b/lib/builtins/arm/chkstk.S index e3002105897e..c5c9ebe0a531 100644 --- a/lib/builtins/arm/chkstk.S +++ b/lib/builtins/arm/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/arm/clzdi2.S b/lib/builtins/arm/clzdi2.S index fc03b385cdfa..685668b11b8e 100644 --- a/lib/builtins/arm/clzdi2.S +++ b/lib/builtins/arm/clzdi2.S @@ -1,16 +1,15 @@ -/* ===-- clzdi2.c - Implement __clzdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements count leading zeros for 64bit arguments. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzdi2.c - Implement __clzdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements count leading zeros for 64bit arguments. +// +//===----------------------------------------------------------------------===// + #include "../assembly.h" .syntax unified @@ -35,14 +34,12 @@ DEFINE_COMPILERRT_FUNCTION(__clzdi2) #endif JMP(lr) #else - /* Assumption: n != 0 */ + // Assumption: n != 0 - /* - * r0: n - * r1: upper half of n, overwritten after check - * r1: count of leading zeros in n + 1 - * r2: scratch register for shifted r0 - */ + // r0: n + // r1: upper half of n, overwritten after check + // r1: count of leading zeros in n + 1 + // r2: scratch register for shifted r0 #ifdef __ARMEB__ cmp r0, 0 moveq r0, r1 @@ -53,14 +50,12 @@ DEFINE_COMPILERRT_FUNCTION(__clzdi2) movne r1, 1 moveq r1, 33 - /* - * Basic block: - * if ((r0 >> SHIFT) == 0) - * r1 += SHIFT; - * else - * r0 >>= SHIFT; - * for descending powers of two as SHIFT. - */ + // Basic block: + // if ((r0 >> SHIFT) == 0) + // r1 += SHIFT; + // else + // r0 >>= SHIFT; + // for descending powers of two as SHIFT. #define BLOCK(shift) \ lsrs r2, r0, shift; \ movne r0, r2; \ @@ -71,18 +66,16 @@ DEFINE_COMPILERRT_FUNCTION(__clzdi2) BLOCK(4) BLOCK(2) - /* - * The basic block invariants at this point are (r0 >> 2) == 0 and - * r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. - * - * r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) - * ---+----------------+----------------+------------+-------------- - * 1 | 1 | 0 | 0 | 1 - * 2 | 0 | 1 | -1 | 0 - * 3 | 0 | 1 | -1 | 0 - * - * The r1's initial value of 1 compensates for the 1 here. - */ + // The basic block invariants at this point are (r0 >> 2) == 0 and + // r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. + // + // r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) + // ---+----------------+----------------+------------+-------------- + // 1 | 1 | 0 | 0 | 1 + // 2 | 0 | 1 | -1 | 0 + // 3 | 0 | 1 | -1 | 0 + // + // The r1's initial value of 1 compensates for the 1 here. sub r0, r1, r0, lsr #1 JMP(lr) diff --git a/lib/builtins/arm/clzsi2.S b/lib/builtins/arm/clzsi2.S index f2ce59c90119..5d86fe486edd 100644 --- a/lib/builtins/arm/clzsi2.S +++ b/lib/builtins/arm/clzsi2.S @@ -1,16 +1,15 @@ -/* ===-- clzsi2.c - Implement __clzsi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements count leading zeros for 32bit arguments. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzsi2.c - Implement __clzsi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements count leading zeros for 32bit arguments. +// +//===----------------------------------------------------------------------===// + #include "../assembly.h" .syntax unified @@ -23,23 +22,19 @@ DEFINE_COMPILERRT_FUNCTION(__clzsi2) clz r0, r0 JMP(lr) #else - /* Assumption: n != 0 */ + // Assumption: n != 0 - /* - * r0: n - * r1: count of leading zeros in n + 1 - * r2: scratch register for shifted r0 - */ + // r0: n + // r1: count of leading zeros in n + 1 + // r2: scratch register for shifted r0 mov r1, 1 - /* - * Basic block: - * if ((r0 >> SHIFT) == 0) - * r1 += SHIFT; - * else - * r0 >>= SHIFT; - * for descending powers of two as SHIFT. - */ + // Basic block: + // if ((r0 >> SHIFT) == 0) + // r1 += SHIFT; + // else + // r0 >>= SHIFT; + // for descending powers of two as SHIFT. #define BLOCK(shift) \ lsrs r2, r0, shift; \ @@ -51,18 +46,16 @@ DEFINE_COMPILERRT_FUNCTION(__clzsi2) BLOCK(4) BLOCK(2) - /* - * The basic block invariants at this point are (r0 >> 2) == 0 and - * r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. - * - * r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) - * ---+----------------+----------------+------------+-------------- - * 1 | 1 | 0 | 0 | 1 - * 2 | 0 | 1 | -1 | 0 - * 3 | 0 | 1 | -1 | 0 - * - * The r1's initial value of 1 compensates for the 1 here. - */ + // The basic block invariants at this point are (r0 >> 2) == 0 and + // r0 != 0. This means 1 <= r0 <= 3 and 0 <= (r0 >> 1) <= 1. + // + // r0 | (r0 >> 1) == 0 | (r0 >> 1) == 1 | -(r0 >> 1) | 1 - (r0 >> 1) + // ---+----------------+----------------+------------+-------------- + // 1 | 1 | 0 | 0 | 1 + // 2 | 0 | 1 | -1 | 0 + // 3 | 0 | 1 | -1 | 0 + // + // The r1's initial value of 1 compensates for the 1 here. sub r0, r1, r0, lsr #1 JMP(lr) diff --git a/lib/builtins/arm/comparesf2.S b/lib/builtins/arm/comparesf2.S index c6c4cc067f07..24b85d2fee15 100644 --- a/lib/builtins/arm/comparesf2.S +++ b/lib/builtins/arm/comparesf2.S @@ -1,9 +1,8 @@ //===-- comparesf2.S - Implement single-precision soft-float comparisons --===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,14 +37,12 @@ //===----------------------------------------------------------------------===// #include "../assembly.h" + .syntax unified .text DEFINE_CODE_STATE -@ int __eqsf2(float a, float b) - - .p2align 2 -DEFINE_COMPILERRT_FUNCTION(__eqsf2) + .macro COMPARESF2_FUNCTION_BODY handle_nan:req #if defined(COMPILER_RT_ARMHF_TARGET) vmov r0, s0 vmov r1, s1 @@ -113,9 +110,9 @@ DEFINE_COMPILERRT_FUNCTION(__eqsf2) // b < 0 ? 1 : -1. Same if a and b have the opposite sign (ignoring Nan). movs r0, #1 lsrs r1, #31 - bne LOCAL_LABEL(CHECK_NAN) + bne LOCAL_LABEL(CHECK_NAN\@) negs r0, r0 - b LOCAL_LABEL(CHECK_NAN) + b LOCAL_LABEL(CHECK_NAN\@) 1: #else it lo @@ -131,7 +128,7 @@ DEFINE_COMPILERRT_FUNCTION(__eqsf2) // Here both have the same sign and absA > absB. movs r0, #1 lsrs r1, #31 - beq LOCAL_LABEL(CHECK_NAN) + beq LOCAL_LABEL(CHECK_NAN\@) negs r0, r0 1: #else @@ -152,7 +149,7 @@ DEFINE_COMPILERRT_FUNCTION(__eqsf2) // Finally, we need to deal with NaNs. If either argument is NaN, replace // the value in r0 with 1. #if defined(USE_THUMB_1) -LOCAL_LABEL(CHECK_NAN): +LOCAL_LABEL(CHECK_NAN\@): movs r6, #0xff lsls r6, #24 cmp r2, r6 @@ -160,92 +157,60 @@ LOCAL_LABEL(CHECK_NAN): cmp r3, r6 1: bls 2f - movs r0, #1 + \handle_nan 2: pop {r6, pc} #else cmp r2, #0xff000000 ite ls cmpls r3, #0xff000000 - movhi r0, #1 + \handle_nan JMP(lr) #endif + .endm + +@ int __eqsf2(float a, float b) + + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__eqsf2) + + .macro __eqsf2_handle_nan +#if defined(USE_THUMB_1) + movs r0, #1 +#else + movhi r0, #1 +#endif + .endm + +COMPARESF2_FUNCTION_BODY __eqsf2_handle_nan + END_COMPILERRT_FUNCTION(__eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__lesf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__ltsf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__nesf2, __eqsf2) +#if defined(__ELF__) +// Alias for libgcc compatibility +DEFINE_COMPILERRT_FUNCTION_ALIAS(__cmpsf2, __lesf2) +#endif + @ int __gtsf2(float a, float b) .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gtsf2) - // Identical to the preceding except in that we return -1 for NaN values. - // Given that the two paths share so much code, one might be tempted to - // unify them; however, the extra code needed to do so makes the code size - // to performance tradeoff very hard to justify for such small functions. -#if defined(COMPILER_RT_ARMHF_TARGET) - vmov r0, s0 - vmov r1, s1 -#endif + + .macro __gtsf2_handle_nan #if defined(USE_THUMB_1) - push {r6, lr} - lsls r2, r0, #1 - lsls r3, r1, #1 - lsrs r6, r3, #1 - orrs r6, r2 - beq 1f - movs r6, r0 - eors r6, r1 -1: - bmi 2f - subs r0, r2, r3 -2: - bhs 3f - movs r0, #1 - lsrs r1, #31 - bne LOCAL_LABEL(CHECK_NAN_2) - negs r0, r0 - b LOCAL_LABEL(CHECK_NAN_2) -3: - bls 4f - movs r0, #1 - lsrs r1, #31 - beq LOCAL_LABEL(CHECK_NAN_2) - negs r0, r0 -4: -LOCAL_LABEL(CHECK_NAN_2): - movs r6, #0xff - lsls r6, #24 - cmp r2, r6 - bhi 5f - cmp r3, r6 -5: - bls 6f movs r0, #1 negs r0, r0 -6: - pop {r6, pc} #else - mov r2, r0, lsl #1 - mov r3, r1, lsl #1 - orrs r12, r2, r3, lsr #1 - it ne - eorsne r12, r0, r1 - it pl - subspl r0, r2, r3 - it lo - mvnlo r0, r1, asr #31 - it hi - movhi r0, r1, asr #31 - it ne - orrne r0, r0, #1 - cmp r2, #0xff000000 - ite ls - cmpls r3, #0xff000000 movhi r0, #-1 - JMP(lr) #endif + .endm + +COMPARESF2_FUNCTION_BODY __gtsf2_handle_nan + END_COMPILERRT_FUNCTION(__gtsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__gesf2, __gtsf2) @@ -283,11 +248,11 @@ DEFINE_COMPILERRT_FUNCTION(__unordsf2) END_COMPILERRT_FUNCTION(__unordsf2) #if defined(COMPILER_RT_ARMHF_TARGET) -DEFINE_COMPILERRT_FUNCTION(__aeabi_fcmpum) +DEFINE_COMPILERRT_FUNCTION(__aeabi_fcmpun) vmov s0, r0 vmov s1, r1 b SYMBOL_NAME(__unordsf2) -END_COMPILERRT_FUNCTION(__aeabi_fcmpum) +END_COMPILERRT_FUNCTION(__aeabi_fcmpun) #else DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_fcmpun, __unordsf2) #endif diff --git a/lib/builtins/arm/divdf3vfp.S b/lib/builtins/arm/divdf3vfp.S index 776ba4f24b47..ad50b57a651d 100644 --- a/lib/builtins/arm/divdf3vfp.S +++ b/lib/builtins/arm/divdf3vfp.S @@ -1,9 +1,8 @@ //===-- divdf3vfp.S - Implement divdf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/divmodsi4.S b/lib/builtins/arm/divmodsi4.S index 8a027b741efe..f94438dfd593 100644 --- a/lib/builtins/arm/divmodsi4.S +++ b/lib/builtins/arm/divmodsi4.S @@ -1,17 +1,16 @@ -/*===-- divmodsi4.S - 32-bit signed integer divide and modulus ------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __divmodsi4 (32-bit signed integer divide and - * modulus) function for the ARM architecture. A naive digit-by-digit - * computation is employed for simplicity. - * - *===----------------------------------------------------------------------===*/ +//===-- divmodsi4.S - 32-bit signed integer divide and modulus ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __divmodsi4 (32-bit signed integer divide and +// modulus) function for the ARM architecture. A naive digit-by-digit +// computation is employed for simplicity. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" diff --git a/lib/builtins/arm/divsf3vfp.S b/lib/builtins/arm/divsf3vfp.S index 130318f0c37b..958a6724bbe3 100644 --- a/lib/builtins/arm/divsf3vfp.S +++ b/lib/builtins/arm/divsf3vfp.S @@ -1,9 +1,8 @@ //===-- divsf3vfp.S - Implement divsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/divsi3.S b/lib/builtins/arm/divsi3.S index 19757af177eb..761bf49d3ed0 100644 --- a/lib/builtins/arm/divsi3.S +++ b/lib/builtins/arm/divsi3.S @@ -1,16 +1,15 @@ -/*===-- divsi3.S - 32-bit signed integer divide ---------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __divsi3 (32-bit signed integer divide) function - * for the ARM architecture as a wrapper around the unsigned routine. - * - *===----------------------------------------------------------------------===*/ +//===-- divsi3.S - 32-bit signed integer divide ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __divsi3 (32-bit signed integer divide) function +// for the ARM architecture as a wrapper around the unsigned routine. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" diff --git a/lib/builtins/arm/eqdf2vfp.S b/lib/builtins/arm/eqdf2vfp.S index d50706570916..2a0a64b97e7d 100644 --- a/lib/builtins/arm/eqdf2vfp.S +++ b/lib/builtins/arm/eqdf2vfp.S @@ -1,21 +1,19 @@ //===-- eqdf2vfp.S - Implement eqdf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // extern int __eqdf2vfp(double a, double b); // // Returns one iff a == b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. -// + .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqdf2vfp) @@ -24,7 +22,7 @@ DEFINE_COMPILERRT_FUNCTION(__eqdf2vfp) #else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register - vcmp.f64 d6, d7 + vcmp.f64 d6, d7 #endif vmrs apsr_nzcv, fpscr ITE(eq) diff --git a/lib/builtins/arm/eqsf2vfp.S b/lib/builtins/arm/eqsf2vfp.S index fd72b2fdbdee..5fefe7b71762 100644 --- a/lib/builtins/arm/eqsf2vfp.S +++ b/lib/builtins/arm/eqsf2vfp.S @@ -1,9 +1,8 @@ //===-- eqsf2vfp.S - Implement eqsf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __eqsf2vfp(float a, float b); // // Returns one iff a == b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/extendsfdf2vfp.S b/lib/builtins/arm/extendsfdf2vfp.S index 1079f977bae6..37c8be8dcd9c 100644 --- a/lib/builtins/arm/extendsfdf2vfp.S +++ b/lib/builtins/arm/extendsfdf2vfp.S @@ -1,9 +1,8 @@ //===-- extendsfdf2vfp.S - Implement extendsfdf2vfp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __extendsfdf2vfp(float a); // // Converts single precision float to double precision result. -// Uses Darwin calling convention where a single precision parameter is +// Uses Darwin calling convention where a single precision parameter is // passed in a GPR and a double precision result is returned in R0/R1 pair. // .syntax unified diff --git a/lib/builtins/arm/fixdfsivfp.S b/lib/builtins/arm/fixdfsivfp.S index 5d7b0f856549..af1d4f4fa5f5 100644 --- a/lib/builtins/arm/fixdfsivfp.S +++ b/lib/builtins/arm/fixdfsivfp.S @@ -1,9 +1,8 @@ //===-- fixdfsivfp.S - Implement fixdfsivfp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __fixdfsivfp(double a); // // Converts double precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a double precision parameter is +// Uses Darwin calling convention where a double precision parameter is // passed in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/fixsfsivfp.S b/lib/builtins/arm/fixsfsivfp.S index 805a277afa34..30b2f3cf9364 100644 --- a/lib/builtins/arm/fixsfsivfp.S +++ b/lib/builtins/arm/fixsfsivfp.S @@ -1,9 +1,8 @@ //===-- fixsfsivfp.S - Implement fixsfsivfp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __fixsfsivfp(float a); // // Converts single precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a single precision parameter is +// Uses Darwin calling convention where a single precision parameter is // passed in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/fixunsdfsivfp.S b/lib/builtins/arm/fixunsdfsivfp.S index 4f1b2c8cefdc..44e6dbd4989e 100644 --- a/lib/builtins/arm/fixunsdfsivfp.S +++ b/lib/builtins/arm/fixunsdfsivfp.S @@ -1,9 +1,8 @@ //===-- fixunsdfsivfp.S - Implement fixunsdfsivfp -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,9 +11,9 @@ // // extern unsigned int __fixunsdfsivfp(double a); // -// Converts double precision float to a 32-bit unsigned int rounding towards +// Converts double precision float to a 32-bit unsigned int rounding towards // zero. All negative values become zero. -// Uses Darwin calling convention where a double precision parameter is +// Uses Darwin calling convention where a double precision parameter is // passed in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/fixunssfsivfp.S b/lib/builtins/arm/fixunssfsivfp.S index e5d778236879..5d6ee7cce27f 100644 --- a/lib/builtins/arm/fixunssfsivfp.S +++ b/lib/builtins/arm/fixunssfsivfp.S @@ -1,9 +1,8 @@ //===-- fixunssfsivfp.S - Implement fixunssfsivfp -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,9 +11,9 @@ // // extern unsigned int __fixunssfsivfp(float a); // -// Converts single precision float to a 32-bit unsigned int rounding towards +// Converts single precision float to a 32-bit unsigned int rounding towards // zero. All negative values become zero. -// Uses Darwin calling convention where a single precision parameter is +// Uses Darwin calling convention where a single precision parameter is // passed in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/floatsidfvfp.S b/lib/builtins/arm/floatsidfvfp.S index 3297ad44d8cd..ae8d2465889c 100644 --- a/lib/builtins/arm/floatsidfvfp.S +++ b/lib/builtins/arm/floatsidfvfp.S @@ -1,9 +1,8 @@ //===-- floatsidfvfp.S - Implement floatsidfvfp ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __floatsidfvfp(int a); // // Converts a 32-bit int to a double precision float. -// Uses Darwin calling convention where a double precision result is +// Uses Darwin calling convention where a double precision result is // return in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/floatsisfvfp.S b/lib/builtins/arm/floatsisfvfp.S index 65408b54b8d4..a36bc5ee5ee0 100644 --- a/lib/builtins/arm/floatsisfvfp.S +++ b/lib/builtins/arm/floatsisfvfp.S @@ -1,9 +1,8 @@ //===-- floatsisfvfp.S - Implement floatsisfvfp ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern float __floatsisfvfp(int a); // // Converts single precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a single precision result is +// Uses Darwin calling convention where a single precision result is // return in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/floatunssidfvfp.S b/lib/builtins/arm/floatunssidfvfp.S index d7a7024a25b8..0932dab2bdb9 100644 --- a/lib/builtins/arm/floatunssidfvfp.S +++ b/lib/builtins/arm/floatunssidfvfp.S @@ -1,9 +1,8 @@ //===-- floatunssidfvfp.S - Implement floatunssidfvfp ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __floatunssidfvfp(unsigned int a); // // Converts a 32-bit int to a double precision float. -// Uses Darwin calling convention where a double precision result is +// Uses Darwin calling convention where a double precision result is // return in GPR register pair. // .syntax unified diff --git a/lib/builtins/arm/floatunssisfvfp.S b/lib/builtins/arm/floatunssisfvfp.S index 1ca856519a92..9578546f40d6 100644 --- a/lib/builtins/arm/floatunssisfvfp.S +++ b/lib/builtins/arm/floatunssisfvfp.S @@ -1,9 +1,8 @@ //===-- floatunssisfvfp.S - Implement floatunssisfvfp ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern float __floatunssisfvfp(unsigned int a); // // Converts single precision float to a 32-bit int rounding towards zero. -// Uses Darwin calling convention where a single precision result is +// Uses Darwin calling convention where a single precision result is // return in a GPR.. // .syntax unified diff --git a/lib/builtins/arm/gedf2vfp.S b/lib/builtins/arm/gedf2vfp.S index 364fc5b24cd1..2af9d909967b 100644 --- a/lib/builtins/arm/gedf2vfp.S +++ b/lib/builtins/arm/gedf2vfp.S @@ -1,9 +1,8 @@ //===-- gedf2vfp.S - Implement gedf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __gedf2vfp(double a, double b); // // Returns one iff a >= b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/gesf2vfp.S b/lib/builtins/arm/gesf2vfp.S index 346c3473ae4c..cedd1e13eb22 100644 --- a/lib/builtins/arm/gesf2vfp.S +++ b/lib/builtins/arm/gesf2vfp.S @@ -1,9 +1,8 @@ //===-- gesf2vfp.S - Implement gesf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __gesf2vfp(float a, float b); // // Returns one iff a >= b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/gtdf2vfp.S b/lib/builtins/arm/gtdf2vfp.S index 3389c3ad9737..782ad8cac013 100644 --- a/lib/builtins/arm/gtdf2vfp.S +++ b/lib/builtins/arm/gtdf2vfp.S @@ -1,9 +1,8 @@ //===-- gtdf2vfp.S - Implement gtdf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __gtdf2vfp(double a, double b); // // Returns one iff a > b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/gtsf2vfp.S b/lib/builtins/arm/gtsf2vfp.S index afdba8b018e2..1cc2bd14f810 100644 --- a/lib/builtins/arm/gtsf2vfp.S +++ b/lib/builtins/arm/gtsf2vfp.S @@ -1,9 +1,8 @@ //===-- gtsf2vfp.S - Implement gtsf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __gtsf2vfp(float a, float b); // // Returns one iff a > b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/ledf2vfp.S b/lib/builtins/arm/ledf2vfp.S index 4bbe4c86837c..0097e4b6c129 100644 --- a/lib/builtins/arm/ledf2vfp.S +++ b/lib/builtins/arm/ledf2vfp.S @@ -1,9 +1,8 @@ //===-- ledf2vfp.S - Implement ledf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __ledf2vfp(double a, double b); // // Returns one iff a <= b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/lesf2vfp.S b/lib/builtins/arm/lesf2vfp.S index 51232bd8cedc..2052d3869145 100644 --- a/lib/builtins/arm/lesf2vfp.S +++ b/lib/builtins/arm/lesf2vfp.S @@ -1,9 +1,8 @@ //===-- lesf2vfp.S - Implement lesf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __lesf2vfp(float a, float b); // // Returns one iff a <= b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/ltdf2vfp.S b/lib/builtins/arm/ltdf2vfp.S index 8e2928c813d2..a126aa9e0536 100644 --- a/lib/builtins/arm/ltdf2vfp.S +++ b/lib/builtins/arm/ltdf2vfp.S @@ -1,9 +1,8 @@ //===-- ltdf2vfp.S - Implement ltdf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern double __ltdf2vfp(double a, double b); // // Returns one iff a < b and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/ltsf2vfp.S b/lib/builtins/arm/ltsf2vfp.S index 59c00c6bab67..ba10d71d2095 100644 --- a/lib/builtins/arm/ltsf2vfp.S +++ b/lib/builtins/arm/ltsf2vfp.S @@ -1,9 +1,8 @@ //===-- ltsf2vfp.S - Implement ltsf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __ltsf2vfp(float a, float b); // // Returns one iff a < b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/modsi3.S b/lib/builtins/arm/modsi3.S index be263834d7f1..5312f5b41c4d 100644 --- a/lib/builtins/arm/modsi3.S +++ b/lib/builtins/arm/modsi3.S @@ -1,16 +1,15 @@ -/*===-- modsi3.S - 32-bit signed integer modulus --------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __modsi3 (32-bit signed integer modulus) function - * for the ARM architecture as a wrapper around the unsigned routine. - * - *===----------------------------------------------------------------------===*/ +//===-- modsi3.S - 32-bit signed integer modulus --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __modsi3 (32-bit signed integer modulus) function +// for the ARM architecture as a wrapper around the unsigned routine. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" diff --git a/lib/builtins/arm/muldf3vfp.S b/lib/builtins/arm/muldf3vfp.S index aa7b23495034..9adc937bcb3f 100644 --- a/lib/builtins/arm/muldf3vfp.S +++ b/lib/builtins/arm/muldf3vfp.S @@ -1,9 +1,8 @@ //===-- muldf3vfp.S - Implement muldf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/mulsf3vfp.S b/lib/builtins/arm/mulsf3vfp.S index a1da789dcade..a94131bebb92 100644 --- a/lib/builtins/arm/mulsf3vfp.S +++ b/lib/builtins/arm/mulsf3vfp.S @@ -1,9 +1,8 @@ //===-- mulsf3vfp.S - Implement mulsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/nedf2vfp.S b/lib/builtins/arm/nedf2vfp.S index aef72eb00974..32d35c41d466 100644 --- a/lib/builtins/arm/nedf2vfp.S +++ b/lib/builtins/arm/nedf2vfp.S @@ -1,21 +1,19 @@ //===-- nedf2vfp.S - Implement nedf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // extern double __nedf2vfp(double a, double b); // // Returns zero if a and b are unequal and neither is NaN. -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. -// + .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__nedf2vfp) @@ -24,7 +22,7 @@ DEFINE_COMPILERRT_FUNCTION(__nedf2vfp) #else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register - vcmp.f64 d6, d7 + vcmp.f64 d6, d7 #endif vmrs apsr_nzcv, fpscr ITE(ne) diff --git a/lib/builtins/arm/negdf2vfp.S b/lib/builtins/arm/negdf2vfp.S index 81f0ab8eec1d..b7cf91877e38 100644 --- a/lib/builtins/arm/negdf2vfp.S +++ b/lib/builtins/arm/negdf2vfp.S @@ -1,9 +1,8 @@ //===-- negdf2vfp.S - Implement negdf2vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,7 +11,7 @@ // // extern double __negdf2vfp(double a, double b); // -// Returns the negation a double precision floating point numbers using the +// Returns the negation a double precision floating point numbers using the // Darwin calling convention where double arguments are passsed in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/negsf2vfp.S b/lib/builtins/arm/negsf2vfp.S index 46ab4a9cf164..b6d3c618927f 100644 --- a/lib/builtins/arm/negsf2vfp.S +++ b/lib/builtins/arm/negsf2vfp.S @@ -1,9 +1,8 @@ //===-- negsf2vfp.S - Implement negsf2vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,7 +11,7 @@ // // extern float __negsf2vfp(float a); // -// Returns the negation of a single precision floating point numbers using the +// Returns the negation of a single precision floating point numbers using the // Darwin calling convention where single arguments are passsed like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/nesf2vfp.S b/lib/builtins/arm/nesf2vfp.S index 50d60f493005..34c8bb4896c2 100644 --- a/lib/builtins/arm/nesf2vfp.S +++ b/lib/builtins/arm/nesf2vfp.S @@ -1,9 +1,8 @@ //===-- nesf2vfp.S - Implement nesf2vfp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __nesf2vfp(float a, float b); // // Returns one iff a != b and neither is NaN. -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/arm/restore_vfp_d8_d15_regs.S b/lib/builtins/arm/restore_vfp_d8_d15_regs.S index 0692cf3e1b77..fd6d59bb3c38 100644 --- a/lib/builtins/arm/restore_vfp_d8_d15_regs.S +++ b/lib/builtins/arm/restore_vfp_d8_d15_regs.S @@ -1,9 +1,8 @@ //===-- save_restore_regs.S - Implement save/restore* ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/save_vfp_d8_d15_regs.S b/lib/builtins/arm/save_vfp_d8_d15_regs.S index 544dd5467a4d..5eb3a2fbaae7 100644 --- a/lib/builtins/arm/save_vfp_d8_d15_regs.S +++ b/lib/builtins/arm/save_vfp_d8_d15_regs.S @@ -1,9 +1,8 @@ //===-- save_restore_regs.S - Implement save/restore* ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/softfloat-alias.list b/lib/builtins/arm/softfloat-alias.list index cc6a4b3cdd2e..ab6ed21e54b0 100644 --- a/lib/builtins/arm/softfloat-alias.list +++ b/lib/builtins/arm/softfloat-alias.list @@ -1,5 +1,5 @@ # -# These are soft float functions which can be +# These are soft float functions which can be # aliased to the *vfp functions on arm processors # that support floating point instructions. # diff --git a/lib/builtins/arm/subdf3vfp.S b/lib/builtins/arm/subdf3vfp.S index 2b6f2bdbfdd5..f4eaf9af1afe 100644 --- a/lib/builtins/arm/subdf3vfp.S +++ b/lib/builtins/arm/subdf3vfp.S @@ -1,9 +1,8 @@ //===-- subdf3vfp.S - Implement subdf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,7 +11,7 @@ // // extern double __subdf3vfp(double a, double b); // -// Returns difference between two double precision floating point numbers using +// Returns difference between two double precision floating point numbers using // the Darwin calling convention where double arguments are passsed in GPR pairs // .syntax unified @@ -23,7 +22,7 @@ DEFINE_COMPILERRT_FUNCTION(__subdf3vfp) #else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vsub.f64 d6, d6, d7 + vsub.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair #endif bx lr diff --git a/lib/builtins/arm/subsf3vfp.S b/lib/builtins/arm/subsf3vfp.S index 3e83ea26507d..80e69f2e8e88 100644 --- a/lib/builtins/arm/subsf3vfp.S +++ b/lib/builtins/arm/subsf3vfp.S @@ -1,9 +1,8 @@ //===-- subsf3vfp.S - Implement subsf3vfp ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switch16.S b/lib/builtins/arm/switch16.S index df9e38e176ce..a4b568da59da 100644 --- a/lib/builtins/arm/switch16.S +++ b/lib/builtins/arm/switch16.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switch32.S b/lib/builtins/arm/switch32.S index d97b5361436d..f2a5af5275cf 100644 --- a/lib/builtins/arm/switch32.S +++ b/lib/builtins/arm/switch32.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switch8.S b/lib/builtins/arm/switch8.S index 4d9e0eaff845..0db875c32914 100644 --- a/lib/builtins/arm/switch8.S +++ b/lib/builtins/arm/switch8.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/switchu8.S b/lib/builtins/arm/switchu8.S index 4ffe35f0549b..551abebc6083 100644 --- a/lib/builtins/arm/switchu8.S +++ b/lib/builtins/arm/switchu8.S @@ -1,9 +1,8 @@ //===-- switch.S - Implement switch* --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/arm/sync-ops.h b/lib/builtins/arm/sync-ops.h index ee02c30c6eaa..c9623249e5d2 100644 --- a/lib/builtins/arm/sync-ops.h +++ b/lib/builtins/arm/sync-ops.h @@ -1,64 +1,61 @@ -/*===-- sync-ops.h - --===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements outline macros for the __sync_fetch_and_* - * operations. Different instantiations will generate appropriate assembly for - * ARM and Thumb-2 versions of the functions. - * - *===----------------------------------------------------------------------===*/ +//===-- sync-ops.h - --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements outline macros for the __sync_fetch_and_* +// operations. Different instantiations will generate appropriate assembly for +// ARM and Thumb-2 versions of the functions. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" -#define SYNC_OP_4(op) \ - .p2align 2 ; \ - .thumb ; \ - .syntax unified ; \ - DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_ ## op) \ - dmb ; \ - mov r12, r0 ; \ - LOCAL_LABEL(tryatomic_ ## op): \ - ldrex r0, [r12] ; \ - op(r2, r0, r1) ; \ - strex r3, r2, [r12] ; \ - cmp r3, #0 ; \ - bne LOCAL_LABEL(tryatomic_ ## op) ; \ - dmb ; \ - bx lr +#define SYNC_OP_4(op) \ + .p2align 2; \ + .thumb; \ + .syntax unified; \ + DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_##op) \ + dmb; \ + mov r12, r0; \ + LOCAL_LABEL(tryatomic_##op) : ldrex r0, [r12]; \ + op(r2, r0, r1); \ + strex r3, r2, [r12]; \ + cmp r3, #0; \ + bne LOCAL_LABEL(tryatomic_##op); \ + dmb; \ + bx lr -#define SYNC_OP_8(op) \ - .p2align 2 ; \ - .thumb ; \ - .syntax unified ; \ - DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_ ## op) \ - push {r4, r5, r6, lr} ; \ - dmb ; \ - mov r12, r0 ; \ - LOCAL_LABEL(tryatomic_ ## op): \ - ldrexd r0, r1, [r12] ; \ - op(r4, r5, r0, r1, r2, r3) ; \ - strexd r6, r4, r5, [r12] ; \ - cmp r6, #0 ; \ - bne LOCAL_LABEL(tryatomic_ ## op) ; \ - dmb ; \ - pop {r4, r5, r6, pc} +#define SYNC_OP_8(op) \ + .p2align 2; \ + .thumb; \ + .syntax unified; \ + DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_##op) \ + push {r4, r5, r6, lr}; \ + dmb; \ + mov r12, r0; \ + LOCAL_LABEL(tryatomic_##op) : ldrexd r0, r1, [r12]; \ + op(r4, r5, r0, r1, r2, r3); \ + strexd r6, r4, r5, [r12]; \ + cmp r6, #0; \ + bne LOCAL_LABEL(tryatomic_##op); \ + dmb; \ + pop { r4, r5, r6, pc } -#define MINMAX_4(rD, rN, rM, cmp_kind) \ - cmp rN, rM ; \ - mov rD, rM ; \ - it cmp_kind ; \ - mov##cmp_kind rD, rN +#define MINMAX_4(rD, rN, rM, cmp_kind) \ + cmp rN, rM; \ + mov rD, rM; \ + it cmp_kind; \ + mov##cmp_kind rD, rN -#define MINMAX_8(rD_LO, rD_HI, rN_LO, rN_HI, rM_LO, rM_HI, cmp_kind) \ - cmp rN_LO, rM_LO ; \ - sbcs rN_HI, rM_HI ; \ - mov rD_LO, rM_LO ; \ - mov rD_HI, rM_HI ; \ - itt cmp_kind ; \ - mov##cmp_kind rD_LO, rN_LO ; \ - mov##cmp_kind rD_HI, rN_HI +#define MINMAX_8(rD_LO, rD_HI, rN_LO, rN_HI, rM_LO, rM_HI, cmp_kind) \ + cmp rN_LO, rM_LO; \ + sbcs rN_HI, rM_HI; \ + mov rD_LO, rM_LO; \ + mov rD_HI, rM_HI; \ + itt cmp_kind; \ + mov##cmp_kind rD_LO, rN_LO; \ + mov##cmp_kind rD_HI, rN_HI diff --git a/lib/builtins/arm/sync_fetch_and_add_4.S b/lib/builtins/arm/sync_fetch_and_add_4.S index 7877d6c46c11..0d55975b7a3a 100644 --- a/lib/builtins/arm/sync_fetch_and_add_4.S +++ b/lib/builtins/arm/sync_fetch_and_add_4.S @@ -1,20 +1,19 @@ -/*===-- sync_fetch_and_add_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_add_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_add_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_add_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" -/* "adds" is 2 bytes shorter than "add". */ +// "adds" is 2 bytes shorter than "add". #define add_4(rD, rN, rM) add rD, rN, rM SYNC_OP_4(add_4) diff --git a/lib/builtins/arm/sync_fetch_and_add_8.S b/lib/builtins/arm/sync_fetch_and_add_8.S index 1df07a342a1b..18bdd875b8b7 100644 --- a/lib/builtins/arm/sync_fetch_and_add_8.S +++ b/lib/builtins/arm/sync_fetch_and_add_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_add_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_add_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_add_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_add_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_and_4.S b/lib/builtins/arm/sync_fetch_and_and_4.S index 720ff02279cd..3a76acca671e 100644 --- a/lib/builtins/arm/sync_fetch_and_and_4.S +++ b/lib/builtins/arm/sync_fetch_and_and_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_and_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_and_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_and_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_and_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_and_8.S b/lib/builtins/arm/sync_fetch_and_and_8.S index 4f7b5ca7ab29..3716eff809d5 100644 --- a/lib/builtins/arm/sync_fetch_and_and_8.S +++ b/lib/builtins/arm/sync_fetch_and_and_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_and_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_and_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_and_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_and_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_max_4.S b/lib/builtins/arm/sync_fetch_and_max_4.S index 43da9c7d4067..b9cee45219de 100644 --- a/lib/builtins/arm/sync_fetch_and_max_4.S +++ b/lib/builtins/arm/sync_fetch_and_max_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_max_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_max_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_max_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_max_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_max_8.S b/lib/builtins/arm/sync_fetch_and_max_8.S index 898fc6202ac8..06115ab55246 100644 --- a/lib/builtins/arm/sync_fetch_and_max_8.S +++ b/lib/builtins/arm/sync_fetch_and_max_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_max_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_max_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_max_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_max_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_min_4.S b/lib/builtins/arm/sync_fetch_and_min_4.S index bba31a03aace..60d435a0bd3f 100644 --- a/lib/builtins/arm/sync_fetch_and_min_4.S +++ b/lib/builtins/arm/sync_fetch_and_min_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_min_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_min_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_min_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_min_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_min_8.S b/lib/builtins/arm/sync_fetch_and_min_8.S index e7ccf9fb60ef..4f3e299d95cc 100644 --- a/lib/builtins/arm/sync_fetch_and_min_8.S +++ b/lib/builtins/arm/sync_fetch_and_min_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_min_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_min_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_min_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_min_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_nand_4.S b/lib/builtins/arm/sync_fetch_and_nand_4.S index c13dd394588f..5a04be0f6455 100644 --- a/lib/builtins/arm/sync_fetch_and_nand_4.S +++ b/lib/builtins/arm/sync_fetch_and_nand_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_nand_4.S - -----------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_nand_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_nand_4.S - -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_nand_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_nand_8.S b/lib/builtins/arm/sync_fetch_and_nand_8.S index e8107ab3a33c..425c94474af7 100644 --- a/lib/builtins/arm/sync_fetch_and_nand_8.S +++ b/lib/builtins/arm/sync_fetch_and_nand_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_nand_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_nand_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_nand_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_nand_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_or_4.S b/lib/builtins/arm/sync_fetch_and_or_4.S index 6726571a944f..f44751b9a9f2 100644 --- a/lib/builtins/arm/sync_fetch_and_or_4.S +++ b/lib/builtins/arm/sync_fetch_and_or_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_or_4.S - -------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_or_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_or_4.S - -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_or_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_or_8.S b/lib/builtins/arm/sync_fetch_and_or_8.S index f7f162c7c3b3..4f18dcf84df9 100644 --- a/lib/builtins/arm/sync_fetch_and_or_8.S +++ b/lib/builtins/arm/sync_fetch_and_or_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_or_8.S - -------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_or_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_or_8.S - -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_or_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_sub_4.S b/lib/builtins/arm/sync_fetch_and_sub_4.S index b9326b14cdd5..999d48c28aec 100644 --- a/lib/builtins/arm/sync_fetch_and_sub_4.S +++ b/lib/builtins/arm/sync_fetch_and_sub_4.S @@ -1,20 +1,19 @@ -/*===-- sync_fetch_and_sub_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_sub_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_sub_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_sub_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" -/* "subs" is 2 bytes shorter than "sub". */ +// "subs" is 2 bytes shorter than "sub". #define sub_4(rD, rN, rM) sub rD, rN, rM SYNC_OP_4(sub_4) diff --git a/lib/builtins/arm/sync_fetch_and_sub_8.S b/lib/builtins/arm/sync_fetch_and_sub_8.S index 6ce743e5ee9f..25a4a1076555 100644 --- a/lib/builtins/arm/sync_fetch_and_sub_8.S +++ b/lib/builtins/arm/sync_fetch_and_sub_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_sub_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_sub_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_sub_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_sub_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umax_4.S b/lib/builtins/arm/sync_fetch_and_umax_4.S index b8d19ff35057..a7b233b15305 100644 --- a/lib/builtins/arm/sync_fetch_and_umax_4.S +++ b/lib/builtins/arm/sync_fetch_and_umax_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umax_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umax_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umax_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umax_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umax_8.S b/lib/builtins/arm/sync_fetch_and_umax_8.S index 34442fd77454..aa5213ff1def 100644 --- a/lib/builtins/arm/sync_fetch_and_umax_8.S +++ b/lib/builtins/arm/sync_fetch_and_umax_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umax_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umax_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umax_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umax_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umin_4.S b/lib/builtins/arm/sync_fetch_and_umin_4.S index 0998e3e10f58..c7a9c89df60a 100644 --- a/lib/builtins/arm/sync_fetch_and_umin_4.S +++ b/lib/builtins/arm/sync_fetch_and_umin_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umin_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umin_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umin_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umin_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_umin_8.S b/lib/builtins/arm/sync_fetch_and_umin_8.S index 558f91390512..8b40541ab47d 100644 --- a/lib/builtins/arm/sync_fetch_and_umin_8.S +++ b/lib/builtins/arm/sync_fetch_and_umin_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_umin_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_umin_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_umin_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_umin_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_xor_4.S b/lib/builtins/arm/sync_fetch_and_xor_4.S index 824f49146880..f509191d0175 100644 --- a/lib/builtins/arm/sync_fetch_and_xor_4.S +++ b/lib/builtins/arm/sync_fetch_and_xor_4.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_xor_4.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_xor_4 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_xor_4.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_xor_4 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_fetch_and_xor_8.S b/lib/builtins/arm/sync_fetch_and_xor_8.S index 073fb9c20f25..7436eb1d4cae 100644 --- a/lib/builtins/arm/sync_fetch_and_xor_8.S +++ b/lib/builtins/arm/sync_fetch_and_xor_8.S @@ -1,16 +1,15 @@ -/*===-- sync_fetch_and_xor_8.S - ------------------------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __sync_fetch_and_xor_8 function for the ARM - * architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- sync_fetch_and_xor_8.S - ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __sync_fetch_and_xor_8 function for the ARM +// architecture. +// +//===----------------------------------------------------------------------===// #include "sync-ops.h" diff --git a/lib/builtins/arm/sync_synchronize.S b/lib/builtins/arm/sync_synchronize.S index 61d1db910f0d..dd06e7191f9b 100644 --- a/lib/builtins/arm/sync_synchronize.S +++ b/lib/builtins/arm/sync_synchronize.S @@ -1,20 +1,17 @@ //===-- sync_synchronize - Implement memory barrier * ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "../assembly.h" -// // When compiling a use of the gcc built-in __sync_synchronize() in thumb1 mode -// the compiler may emit a call to __sync_synchronize. -// On Darwin the implementation jumps to an OS supplied function named +// the compiler may emit a call to __sync_synchronize. +// On Darwin the implementation jumps to an OS supplied function named // OSMemoryBarrier -// .text .syntax unified @@ -31,7 +28,7 @@ END_COMPILERRT_FUNCTION(__sync_synchronize) // tell linker it can break up file at label boundaries .subsections_via_symbols - + #endif NO_EXEC_STACK_DIRECTIVE diff --git a/lib/builtins/arm/truncdfsf2vfp.S b/lib/builtins/arm/truncdfsf2vfp.S index 682e54d3d294..a3c0a73466e9 100644 --- a/lib/builtins/arm/truncdfsf2vfp.S +++ b/lib/builtins/arm/truncdfsf2vfp.S @@ -1,9 +1,8 @@ //===-- truncdfsf2vfp.S - Implement truncdfsf2vfp -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern float __truncdfsf2vfp(double a); // // Converts double precision float to signle precision result. -// Uses Darwin calling convention where a double precision parameter is +// Uses Darwin calling convention where a double precision parameter is // passed in a R0/R1 pair and a signle precision result is returned in R0. // .syntax unified diff --git a/lib/builtins/arm/udivmodsi4.S b/lib/builtins/arm/udivmodsi4.S index ee3950c9b0eb..0f40575fe99e 100644 --- a/lib/builtins/arm/udivmodsi4.S +++ b/lib/builtins/arm/udivmodsi4.S @@ -1,16 +1,15 @@ -/*===-- udivmodsi4.S - 32-bit unsigned integer divide and modulus ---------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __udivmodsi4 (32-bit unsigned integer divide and - * modulus) function for the ARM 32-bit architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- udivmodsi4.S - 32-bit unsigned integer divide and modulus ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __udivmodsi4 (32-bit unsigned integer divide and +// modulus) function for the ARM 32-bit architecture. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" @@ -39,26 +38,25 @@ DEFINE_COMPILERRT_FUNCTION(__udivmodsi4) beq LOCAL_LABEL(divby1) cmp r0, r1 bcc LOCAL_LABEL(quotient0) - /* - * Implement division using binary long division algorithm. - * - * r0 is the numerator, r1 the denominator. - * - * The code before JMP computes the correct shift I, so that - * r0 and (r1 << I) have the highest bit set in the same position. - * At the time of JMP, ip := .Ldiv0block - 12 * I. - * This depends on the fixed instruction size of block. - * For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. - * - * block(shift) implements the test-and-update-quotient core. - * It assumes (r0 << shift) can be computed without overflow and - * that (r0 << shift) < 2 * r1. The quotient is stored in r3. - */ + + // Implement division using binary long division algorithm. + // + // r0 is the numerator, r1 the denominator. + // + // The code before JMP computes the correct shift I, so that + // r0 and (r1 << I) have the highest bit set in the same position. + // At the time of JMP, ip := .Ldiv0block - 12 * I. + // This depends on the fixed instruction size of block. + // For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. + // + // block(shift) implements the test-and-update-quotient core. + // It assumes (r0 << shift) can be computed without overflow and + // that (r0 << shift) < 2 * r1. The quotient is stored in r3. # ifdef __ARM_FEATURE_CLZ clz ip, r0 clz r3, r1 - /* r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. */ + // r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. sub r3, r3, ip # if defined(USE_THUMB_2) adr ip, LOCAL_LABEL(div0block) + 1 @@ -99,11 +97,11 @@ DEFINE_COMPILERRT_FUNCTION(__udivmodsi4) movhs r4, r3 subhs ip, ip, #(2 * 12) - /* Last block, no need to update r3 or r4. */ + // Last block, no need to update r3 or r4. cmp r1, r4, lsr #1 subls ip, ip, #(1 * 12) - ldr r4, [sp], #8 /* restore r4, we are done with it. */ + ldr r4, [sp], #8 // restore r4, we are done with it. mov r3, #0 JMP(ip) @@ -164,7 +162,7 @@ LOCAL_LABEL(divby1): mov r3, #0 str r3, [r2] JMP(lr) -#endif /* __ARM_ARCH_EXT_IDIV__ */ +#endif // __ARM_ARCH_EXT_IDIV__ LOCAL_LABEL(divby0): mov r0, #0 diff --git a/lib/builtins/arm/udivsi3.S b/lib/builtins/arm/udivsi3.S index 6dea27d404ff..9b1b035b33d6 100644 --- a/lib/builtins/arm/udivsi3.S +++ b/lib/builtins/arm/udivsi3.S @@ -1,16 +1,15 @@ -/*===-- udivsi3.S - 32-bit unsigned integer divide ------------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __udivsi3 (32-bit unsigned integer divide) - * function for the ARM 32-bit architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- udivsi3.S - 32-bit unsigned integer divide ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __udivsi3 (32-bit unsigned integer divide) +// function for the ARM 32-bit architecture. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" @@ -40,7 +39,7 @@ LOCAL_LABEL(divby0): JMP(lr) # endif -#else /* ! __ARM_ARCH_EXT_IDIV__ */ +#else // ! __ARM_ARCH_EXT_IDIV__ cmp r1, #1 bcc LOCAL_LABEL(divby0) #if defined(USE_THUMB_1) @@ -63,26 +62,24 @@ LOCAL_LABEL(num_ge_denom): JMPc(lr, cc) #endif - /* - * Implement division using binary long division algorithm. - * - * r0 is the numerator, r1 the denominator. - * - * The code before JMP computes the correct shift I, so that - * r0 and (r1 << I) have the highest bit set in the same position. - * At the time of JMP, ip := .Ldiv0block - 12 * I. - * This depends on the fixed instruction size of block. - * For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. - * - * block(shift) implements the test-and-update-quotient core. - * It assumes (r0 << shift) can be computed without overflow and - * that (r0 << shift) < 2 * r1. The quotient is stored in r3. - */ + // Implement division using binary long division algorithm. + // + // r0 is the numerator, r1 the denominator. + // + // The code before JMP computes the correct shift I, so that + // r0 and (r1 << I) have the highest bit set in the same position. + // At the time of JMP, ip := .Ldiv0block - 12 * I. + // This depends on the fixed instruction size of block. + // For ARM mode, this is 12 Bytes, for THUMB mode 14 Bytes. + // + // block(shift) implements the test-and-update-quotient core. + // It assumes (r0 << shift) can be computed without overflow and + // that (r0 << shift) < 2 * r1. The quotient is stored in r3. # if defined(__ARM_FEATURE_CLZ) clz ip, r0 clz r3, r1 - /* r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. */ + // r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. sub r3, r3, ip # if defined(USE_THUMB_2) adr ip, LOCAL_LABEL(div0block) + 1 @@ -94,7 +91,7 @@ LOCAL_LABEL(num_ge_denom): sub ip, ip, r3, lsl #3 mov r3, #0 bx ip -# else /* No CLZ Feature */ +# else // No CLZ Feature # if defined(USE_THUMB_2) # error THUMB mode requires CLZ or UDIV # endif @@ -160,7 +157,7 @@ LOCAL_LABEL(skip_2): subhs ip, ip, #(2 * BLOCK_SIZE) # endif - /* Last block, no need to update r2 or r3. */ + // Last block, no need to update r2 or r3. # if defined(USE_THUMB_1) lsrs r3, r2, #1 cmp r3, r1 @@ -180,12 +177,12 @@ LOCAL_LABEL(skip_1): JMP(ip) # endif -# endif /* __ARM_FEATURE_CLZ */ +# endif // __ARM_FEATURE_CLZ #define IMM # - /* due to the range limit of branch in Thumb1, we have to place the - block closer */ + // due to the range limit of branch in Thumb1, we have to place the + // block closer LOCAL_LABEL(divby0): movs r0, #0 # if defined(__ARM_EABI__) @@ -204,13 +201,13 @@ LOCAL_LABEL(divby0): blo LOCAL_LABEL(block_skip_##shift); \ subs r0, r0, r2; \ LOCAL_LABEL(block_skip_##shift) :; \ - adcs r3, r3 /* same as ((r3 << 1) | Carry). Carry is set if r0 >= r2. */ + adcs r3, r3 // same as ((r3 << 1) | Carry). Carry is set if r0 >= r2. - /* TODO: if current location counter is not not word aligned, we don't - need the .p2align and nop */ - /* Label div0block must be word-aligned. First align block 31 */ + // TODO: if current location counter is not not word aligned, we don't + // need the .p2align and nop + // Label div0block must be word-aligned. First align block 31 .p2align 2 - nop /* Padding to align div0block as 31 blocks = 310 bytes */ + nop // Padding to align div0block as 31 blocks = 310 bytes #else #define block(shift) \ @@ -256,7 +253,7 @@ LOCAL_LABEL(div0block): mov r0, r3 JMP(lr) -#endif /* __ARM_ARCH_EXT_IDIV__ */ +#endif // __ARM_ARCH_EXT_IDIV__ END_COMPILERRT_FUNCTION(__udivsi3) diff --git a/lib/builtins/arm/umodsi3.S b/lib/builtins/arm/umodsi3.S index 069fad34cb9c..5ab78de17853 100644 --- a/lib/builtins/arm/umodsi3.S +++ b/lib/builtins/arm/umodsi3.S @@ -1,16 +1,15 @@ -/*===-- umodsi3.S - 32-bit unsigned integer modulus -----------------------===// - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------===// - * - * This file implements the __umodsi3 (32-bit unsigned integer modulus) - * function for the ARM 32-bit architecture. - * - *===----------------------------------------------------------------------===*/ +//===-- umodsi3.S - 32-bit unsigned integer modulus -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the __umodsi3 (32-bit unsigned integer modulus) +// function for the ARM 32-bit architecture. +// +//===----------------------------------------------------------------------===// #include "../assembly.h" @@ -38,26 +37,25 @@ DEFINE_COMPILERRT_FUNCTION(__umodsi3) cmp r0, r1 IT(cc) JMPc(lr, cc) - /* - * Implement division using binary long division algorithm. - * - * r0 is the numerator, r1 the denominator. - * - * The code before JMP computes the correct shift I, so that - * r0 and (r1 << I) have the highest bit set in the same position. - * At the time of JMP, ip := .Ldiv0block - 8 * I. - * This depends on the fixed instruction size of block. - * For ARM mode, this is 8 Bytes, for THUMB mode 10 Bytes. - * - * block(shift) implements the test-and-update-quotient core. - * It assumes (r0 << shift) can be computed without overflow and - * that (r0 << shift) < 2 * r1. The quotient is stored in r3. - */ + + // Implement division using binary long division algorithm. + // + // r0 is the numerator, r1 the denominator. + // + // The code before JMP computes the correct shift I, so that + // r0 and (r1 << I) have the highest bit set in the same position. + // At the time of JMP, ip := .Ldiv0block - 8 * I. + // This depends on the fixed instruction size of block. + // For ARM mode, this is 8 Bytes, for THUMB mode 10 Bytes. + // + // block(shift) implements the test-and-update-quotient core. + // It assumes (r0 << shift) can be computed without overflow and + // that (r0 << shift) < 2 * r1. The quotient is stored in r3. # ifdef __ARM_FEATURE_CLZ clz ip, r0 clz r3, r1 - /* r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. */ + // r0 >= r1 implies clz(r0) <= clz(r1), so ip <= r3. sub r3, r3, ip # if defined(USE_THUMB_2) adr ip, LOCAL_LABEL(div0block) + 1 @@ -94,7 +92,7 @@ DEFINE_COMPILERRT_FUNCTION(__umodsi3) movhs r2, r3 subhs ip, ip, #(2 * 8) - /* Last block, no need to update r2 or r3. */ + // Last block, no need to update r2 or r3. cmp r1, r2, lsr #1 subls ip, ip, #(1 * 8) @@ -142,7 +140,7 @@ DEFINE_COMPILERRT_FUNCTION(__umodsi3) LOCAL_LABEL(div0block): block(0) JMP(lr) -#endif /* __ARM_ARCH_EXT_IDIV__ */ +#endif // __ARM_ARCH_EXT_IDIV__ LOCAL_LABEL(divby0): mov r0, #0 diff --git a/lib/builtins/arm/unorddf2vfp.S b/lib/builtins/arm/unorddf2vfp.S index 6625fa8a3119..ea36a1cb5594 100644 --- a/lib/builtins/arm/unorddf2vfp.S +++ b/lib/builtins/arm/unorddf2vfp.S @@ -1,9 +1,8 @@ //===-- unorddf2vfp.S - Implement unorddf2vfp ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __unorddf2vfp(double a, double b); // // Returns one iff a or b is NaN -// Uses Darwin calling convention where double precision arguments are passsed +// Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified diff --git a/lib/builtins/arm/unordsf2vfp.S b/lib/builtins/arm/unordsf2vfp.S index 0b5da2ba3e17..7311297329dc 100644 --- a/lib/builtins/arm/unordsf2vfp.S +++ b/lib/builtins/arm/unordsf2vfp.S @@ -1,9 +1,8 @@ //===-- unordsf2vfp.S - Implement unordsf2vfp -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,7 @@ // extern int __unordsf2vfp(float a, float b); // // Returns one iff a or b is NaN -// Uses Darwin calling convention where single precision arguments are passsed +// Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified diff --git a/lib/builtins/ashldi3.c b/lib/builtins/ashldi3.c index a5c1836006b9..7c81057a2284 100644 --- a/lib/builtins/ashldi3.c +++ b/lib/builtins/ashldi3.c @@ -1,45 +1,38 @@ -/* ====-- ashldi3.c - Implement __ashldi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashldi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +// ====-- ashldi3.c - Implement __ashldi3 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashldi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a << b */ +// Returns: a << b -/* Precondition: 0 <= b < bits_in_dword */ +// Precondition: 0 <= b < bits_in_dword -COMPILER_RT_ABI di_int -__ashldi3(di_int a, si_int b) -{ - const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); - dwords input; - dwords result; - input.all = a; - if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ - { - result.s.low = 0; - result.s.high = input.s.low << (b - bits_in_word); - } - else /* 0 <= b < bits_in_word */ - { - if (b == 0) - return a; - result.s.low = input.s.low << b; - result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_word - b)); - } - return result.all; +COMPILER_RT_ABI di_int __ashldi3(di_int a, si_int b) { + const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); + dwords input; + dwords result; + input.all = a; + if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ { + result.s.low = 0; + result.s.high = input.s.low << (b - bits_in_word); + } else /* 0 <= b < bits_in_word */ { + if (b == 0) + return a; + result.s.low = input.s.low << b; + result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_word - b)); + } + return result.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) COMPILER_RT_ALIAS(__ashldi3); +COMPILER_RT_ALIAS(__ashldi3, __aeabi_llsl) #endif diff --git a/lib/builtins/ashlti3.c b/lib/builtins/ashlti3.c index 638ae845ff04..2d7bd4a89380 100644 --- a/lib/builtins/ashlti3.c +++ b/lib/builtins/ashlti3.c @@ -1,45 +1,38 @@ -/* ===-- ashlti3.c - Implement __ashlti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashlti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ashlti3.c - Implement __ashlti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashlti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a << b */ +// Returns: a << b -/* Precondition: 0 <= b < bits_in_tword */ +// Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int -__ashlti3(ti_int a, si_int b) -{ - const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); - twords input; - twords result; - input.all = a; - if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ - { - result.s.low = 0; - result.s.high = input.s.low << (b - bits_in_dword); - } - else /* 0 <= b < bits_in_dword */ - { - if (b == 0) - return a; - result.s.low = input.s.low << b; - result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_dword - b)); - } - return result.all; +COMPILER_RT_ABI ti_int __ashlti3(ti_int a, si_int b) { + const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); + twords input; + twords result; + input.all = a; + if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ { + result.s.low = 0; + result.s.high = input.s.low << (b - bits_in_dword); + } else /* 0 <= b < bits_in_dword */ { + if (b == 0) + return a; + result.s.low = input.s.low << b; + result.s.high = (input.s.high << b) | (input.s.low >> (bits_in_dword - b)); + } + return result.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/ashrdi3.c b/lib/builtins/ashrdi3.c index 84619965eca0..b9939132205c 100644 --- a/lib/builtins/ashrdi3.c +++ b/lib/builtins/ashrdi3.c @@ -1,46 +1,39 @@ -/*===-- ashrdi3.c - Implement __ashrdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashrdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ashrdi3.c - Implement __ashrdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashrdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: arithmetic a >> b */ +// Returns: arithmetic a >> b -/* Precondition: 0 <= b < bits_in_dword */ +// Precondition: 0 <= b < bits_in_dword -COMPILER_RT_ABI di_int -__ashrdi3(di_int a, si_int b) -{ - const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); - dwords input; - dwords result; - input.all = a; - if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ - { - /* result.s.high = input.s.high < 0 ? -1 : 0 */ - result.s.high = input.s.high >> (bits_in_word - 1); - result.s.low = input.s.high >> (b - bits_in_word); - } - else /* 0 <= b < bits_in_word */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI di_int __ashrdi3(di_int a, si_int b) { + const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); + dwords input; + dwords result; + input.all = a; + if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ { + // result.s.high = input.s.high < 0 ? -1 : 0 + result.s.high = input.s.high >> (bits_in_word - 1); + result.s.low = input.s.high >> (b - bits_in_word); + } else /* 0 <= b < bits_in_word */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); + } + return result.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) COMPILER_RT_ALIAS(__ashrdi3); +COMPILER_RT_ALIAS(__ashrdi3, __aeabi_lasr) #endif diff --git a/lib/builtins/ashrti3.c b/lib/builtins/ashrti3.c index f78205d961e4..f573b6d6ccba 100644 --- a/lib/builtins/ashrti3.c +++ b/lib/builtins/ashrti3.c @@ -1,46 +1,39 @@ -/* ===-- ashrti3.c - Implement __ashrti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ashrti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ashrti3.c - Implement __ashrti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ashrti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: arithmetic a >> b */ +// Returns: arithmetic a >> b -/* Precondition: 0 <= b < bits_in_tword */ +// Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int -__ashrti3(ti_int a, si_int b) -{ - const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); - twords input; - twords result; - input.all = a; - if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ - { - /* result.s.high = input.s.high < 0 ? -1 : 0 */ - result.s.high = input.s.high >> (bits_in_dword - 1); - result.s.low = input.s.high >> (b - bits_in_dword); - } - else /* 0 <= b < bits_in_dword */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI ti_int __ashrti3(ti_int a, si_int b) { + const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); + twords input; + twords result; + input.all = a; + if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ { + // result.s.high = input.s.high < 0 ? -1 : 0 + result.s.high = input.s.high >> (bits_in_dword - 1); + result.s.low = input.s.high >> (b - bits_in_dword); + } else /* 0 <= b < bits_in_dword */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); + } + return result.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/assembly.h b/lib/builtins/assembly.h index 3f5e59b25442..f437cb87f60a 100644 --- a/lib/builtins/assembly.h +++ b/lib/builtins/assembly.h @@ -1,17 +1,15 @@ -/* ===-- assembly.h - compiler-rt assembler support macros -----------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file defines macros for use in compiler-rt assembler source. - * This file is not part of the interface of this library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- assembly.h - compiler-rt assembler support macros -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines macros for use in compiler-rt assembler source. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// #ifndef COMPILERRT_ASSEMBLY_H #define COMPILERRT_ASSEMBLY_H @@ -69,11 +67,9 @@ #if defined(__arm__) -/* - * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: - * - for '-mthumb -march=armv6' compiler defines '__thumb__' - * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' - */ +// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: +// - for '-mthumb -march=armv6' compiler defines '__thumb__' +// - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' #if defined(__thumb2__) || defined(__thumb__) #define DEFINE_CODE_STATE .thumb SEPARATOR #define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR @@ -201,4 +197,4 @@ #define END_COMPILERRT_FUNCTION(name) #endif -#endif /* COMPILERRT_ASSEMBLY_H */ +#endif // COMPILERRT_ASSEMBLY_H diff --git a/lib/builtins/atomic.c b/lib/builtins/atomic.c index ee35e342eda5..0f82803a6416 100644 --- a/lib/builtins/atomic.c +++ b/lib/builtins/atomic.c @@ -1,29 +1,27 @@ -/*===-- atomic.c - Implement support functions for atomic operations.------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * atomic.c defines a set of functions for performing atomic accesses on - * arbitrary-sized memory locations. This design uses locks that should - * be fast in the uncontended case, for two reasons: - * - * 1) This code must work with C programs that do not link to anything - * (including pthreads) and so it should not depend on any pthread - * functions. - * 2) Atomic operations, rather than explicit mutexes, are most commonly used - * on code where contended operations are rate. - * - * To avoid needing a per-object lock, this code allocates an array of - * locks and hashes the object pointers to find the one that it should use. - * For operations that must be atomic on two locations, the lower lock is - * always acquired first, to avoid deadlock. - * - *===----------------------------------------------------------------------=== - */ +//===-- atomic.c - Implement support functions for atomic operations.------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// atomic.c defines a set of functions for performing atomic accesses on +// arbitrary-sized memory locations. This design uses locks that should +// be fast in the uncontended case, for two reasons: +// +// 1) This code must work with C programs that do not link to anything +// (including pthreads) and so it should not depend on any pthread +// functions. +// 2) Atomic operations, rather than explicit mutexes, are most commonly used +// on code where contended operations are rate. +// +// To avoid needing a per-object lock, this code allocates an array of +// locks and hashes the object pointers to find the one that it should use. +// For operations that must be atomic on two locations, the lower lock is +// always acquired first, to avoid deadlock. +// +//===----------------------------------------------------------------------===// #include <stdint.h> #include <string.h> @@ -35,13 +33,14 @@ #pragma redefine_extname __atomic_load_c SYMBOL_NAME(__atomic_load) #pragma redefine_extname __atomic_store_c SYMBOL_NAME(__atomic_store) #pragma redefine_extname __atomic_exchange_c SYMBOL_NAME(__atomic_exchange) -#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME(__atomic_compare_exchange) +#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME( \ + __atomic_compare_exchange) /// Number of locks. This allocates one page on 32-bit platforms, two on /// 64-bit. This can be specified externally if a different trade between /// memory usage and contention probability is required for a given platform. #ifndef SPINLOCK_COUNT -#define SPINLOCK_COUNT (1<<10) +#define SPINLOCK_COUNT (1 << 10) #endif static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; @@ -52,38 +51,35 @@ static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; //////////////////////////////////////////////////////////////////////////////// #ifdef __FreeBSD__ #include <errno.h> -#include <sys/types.h> #include <machine/atomic.h> +#include <sys/types.h> #include <sys/umtx.h> typedef struct _usem Lock; __inline static void unlock(Lock *l) { - __c11_atomic_store((_Atomic(uint32_t)*)&l->_count, 1, __ATOMIC_RELEASE); + __c11_atomic_store((_Atomic(uint32_t) *)&l->_count, 1, __ATOMIC_RELEASE); __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); if (l->_has_waiters) - _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); + _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); } __inline static void lock(Lock *l) { uint32_t old = 1; - while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t)*)&l->_count, &old, - 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { + while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t) *)&l->_count, + &old, 0, __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) { _umtx_op(l, UMTX_OP_SEM_WAIT, 0, 0, 0); old = 1; } } /// locks for atomic operations -static Lock locks[SPINLOCK_COUNT] = { [0 ... SPINLOCK_COUNT-1] = {0,1,0} }; +static Lock locks[SPINLOCK_COUNT] = {[0 ... SPINLOCK_COUNT - 1] = {0, 1, 0}}; #elif defined(__APPLE__) #include <libkern/OSAtomic.h> typedef OSSpinLock Lock; -__inline static void unlock(Lock *l) { - OSSpinLockUnlock(l); -} +__inline static void unlock(Lock *l) { OSSpinLockUnlock(l); } /// Locks a lock. In the current implementation, this is potentially /// unbounded in the contended case. -__inline static void lock(Lock *l) { - OSSpinLockLock(l); -} +__inline static void lock(Lock *l) { OSSpinLockLock(l); } static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 #else @@ -97,20 +93,19 @@ __inline static void unlock(Lock *l) { __inline static void lock(Lock *l) { uintptr_t old = 0; while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED)) + __ATOMIC_RELAXED)) old = 0; } /// locks for atomic operations static Lock locks[SPINLOCK_COUNT]; #endif - -/// Returns a lock to use for a given pointer. +/// Returns a lock to use for a given pointer. static __inline Lock *lock_for_pointer(void *ptr) { intptr_t hash = (intptr_t)ptr; // Disregard the lowest 4 bits. We want all values that may be part of the // same memory operation to hash to the same value and therefore use the same - // lock. + // lock. hash >>= 4; // Use the next bits as the basis for the hash intptr_t low = hash & SPINLOCK_MASK; @@ -133,36 +128,44 @@ static __inline Lock *lock_for_pointer(void *ptr) { /// Macro that calls the compiler-generated lock-free versions of functions /// when they exist. -#define LOCK_FREE_CASES() \ - do {\ - switch (size) {\ - case 2:\ - if (IS_LOCK_FREE_2) {\ - LOCK_FREE_ACTION(uint16_t);\ - }\ - case 4:\ - if (IS_LOCK_FREE_4) {\ - LOCK_FREE_ACTION(uint32_t);\ - }\ - case 8:\ - if (IS_LOCK_FREE_8) {\ - LOCK_FREE_ACTION(uint64_t);\ - }\ - case 16:\ - if (IS_LOCK_FREE_16) {\ - /* FIXME: __uint128_t isn't available on 32 bit platforms. - LOCK_FREE_ACTION(__uint128_t);*/\ - }\ - }\ +#define LOCK_FREE_CASES() \ + do { \ + switch (size) { \ + case 1: \ + if (IS_LOCK_FREE_1) { \ + LOCK_FREE_ACTION(uint8_t); \ + } \ + break; \ + case 2: \ + if (IS_LOCK_FREE_2) { \ + LOCK_FREE_ACTION(uint16_t); \ + } \ + break; \ + case 4: \ + if (IS_LOCK_FREE_4) { \ + LOCK_FREE_ACTION(uint32_t); \ + } \ + break; \ + case 8: \ + if (IS_LOCK_FREE_8) { \ + LOCK_FREE_ACTION(uint64_t); \ + } \ + break; \ + case 16: \ + if (IS_LOCK_FREE_16) { \ + /* FIXME: __uint128_t isn't available on 32 bit platforms. \ + LOCK_FREE_ACTION(__uint128_t);*/ \ + } \ + break; \ + } \ } while (0) - /// An atomic load operation. This is atomic with respect to the source /// pointer only. void __atomic_load_c(int size, void *src, void *dest, int model) { -#define LOCK_FREE_ACTION(type) \ - *((type*)dest) = __c11_atomic_load((_Atomic(type)*)src, model);\ - return; +#define LOCK_FREE_ACTION(type) \ + *((type *)dest) = __c11_atomic_load((_Atomic(type) *)src, model); \ + return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(src); @@ -174,9 +177,9 @@ void __atomic_load_c(int size, void *src, void *dest, int model) { /// An atomic store operation. This is atomic with respect to the destination /// pointer only. void __atomic_store_c(int size, void *dest, void *src, int model) { -#define LOCK_FREE_ACTION(type) \ - __c11_atomic_store((_Atomic(type)*)dest, *(type*)dest, model);\ - return; +#define LOCK_FREE_ACTION(type) \ + __c11_atomic_store((_Atomic(type) *)dest, *(type *)src, model); \ + return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(dest); @@ -189,12 +192,13 @@ void __atomic_store_c(int size, void *dest, void *src, int model) { /// to the value at *expected, then this copies value at *desired to *ptr. If /// they are not, then this stores the current value from *ptr in *expected. /// -/// This function returns 1 if the exchange takes place or 0 if it fails. +/// This function returns 1 if the exchange takes place or 0 if it fails. int __atomic_compare_exchange_c(int size, void *ptr, void *expected, - void *desired, int success, int failure) { -#define LOCK_FREE_ACTION(type) \ - return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, (type*)expected,\ - *(type*)desired, success, failure) + void *desired, int success, int failure) { +#define LOCK_FREE_ACTION(type) \ + return __c11_atomic_compare_exchange_strong( \ + (_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \ + failure) LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); @@ -212,10 +216,10 @@ int __atomic_compare_exchange_c(int size, void *ptr, void *expected, /// Performs an atomic exchange operation between two pointers. This is atomic /// with respect to the target address. void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) { -#define LOCK_FREE_ACTION(type) \ - *(type*)old = __c11_atomic_exchange((_Atomic(type)*)ptr, *(type*)val,\ - model);\ - return; +#define LOCK_FREE_ACTION(type) \ + *(type *)old = \ + __c11_atomic_exchange((_Atomic(type) *)ptr, *(type *)val, model); \ + return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); @@ -230,96 +234,96 @@ void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) { // specialised versions of the above functions. //////////////////////////////////////////////////////////////////////////////// #ifdef __SIZEOF_INT128__ -#define OPTIMISED_CASES\ - OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t)\ - OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t)\ - OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t)\ - OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t)\ +#define OPTIMISED_CASES \ + OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ + OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ + OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ + OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) \ OPTIMISED_CASE(16, IS_LOCK_FREE_16, __uint128_t) #else -#define OPTIMISED_CASES\ - OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t)\ - OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t)\ - OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t)\ +#define OPTIMISED_CASES \ + OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ + OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ + OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) #endif -#define OPTIMISED_CASE(n, lockfree, type)\ -type __atomic_load_##n(type *src, int model) {\ - if (lockfree)\ - return __c11_atomic_load((_Atomic(type)*)src, model);\ - Lock *l = lock_for_pointer(src);\ - lock(l);\ - type val = *src;\ - unlock(l);\ - return val;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + type __atomic_load_##n(type *src, int model) { \ + if (lockfree) \ + return __c11_atomic_load((_Atomic(type) *)src, model); \ + Lock *l = lock_for_pointer(src); \ + lock(l); \ + type val = *src; \ + unlock(l); \ + return val; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE -#define OPTIMISED_CASE(n, lockfree, type)\ -void __atomic_store_##n(type *dest, type val, int model) {\ - if (lockfree) {\ - __c11_atomic_store((_Atomic(type)*)dest, val, model);\ - return;\ - }\ - Lock *l = lock_for_pointer(dest);\ - lock(l);\ - *dest = val;\ - unlock(l);\ - return;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + void __atomic_store_##n(type *dest, type val, int model) { \ + if (lockfree) { \ + __c11_atomic_store((_Atomic(type) *)dest, val, model); \ + return; \ + } \ + Lock *l = lock_for_pointer(dest); \ + lock(l); \ + *dest = val; \ + unlock(l); \ + return; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE -#define OPTIMISED_CASE(n, lockfree, type)\ -type __atomic_exchange_##n(type *dest, type val, int model) {\ - if (lockfree)\ - return __c11_atomic_exchange((_Atomic(type)*)dest, val, model);\ - Lock *l = lock_for_pointer(dest);\ - lock(l);\ - type tmp = *dest;\ - *dest = val;\ - unlock(l);\ - return tmp;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + type __atomic_exchange_##n(type *dest, type val, int model) { \ + if (lockfree) \ + return __c11_atomic_exchange((_Atomic(type) *)dest, val, model); \ + Lock *l = lock_for_pointer(dest); \ + lock(l); \ + type tmp = *dest; \ + *dest = val; \ + unlock(l); \ + return tmp; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE -#define OPTIMISED_CASE(n, lockfree, type)\ -int __atomic_compare_exchange_##n(type *ptr, type *expected, type desired,\ - int success, int failure) {\ - if (lockfree)\ - return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, expected, desired,\ - success, failure);\ - Lock *l = lock_for_pointer(ptr);\ - lock(l);\ - if (*ptr == *expected) {\ - *ptr = desired;\ - unlock(l);\ - return 1;\ - }\ - *expected = *ptr;\ - unlock(l);\ - return 0;\ -} +#define OPTIMISED_CASE(n, lockfree, type) \ + int __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \ + int success, int failure) { \ + if (lockfree) \ + return __c11_atomic_compare_exchange_strong( \ + (_Atomic(type) *)ptr, expected, desired, success, failure); \ + Lock *l = lock_for_pointer(ptr); \ + lock(l); \ + if (*ptr == *expected) { \ + *ptr = desired; \ + unlock(l); \ + return 1; \ + } \ + *expected = *ptr; \ + unlock(l); \ + return 0; \ + } OPTIMISED_CASES #undef OPTIMISED_CASE //////////////////////////////////////////////////////////////////////////////// // Atomic read-modify-write operations for integers of various sizes. //////////////////////////////////////////////////////////////////////////////// -#define ATOMIC_RMW(n, lockfree, type, opname, op) \ -type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) {\ - if (lockfree) \ - return __c11_atomic_fetch_##opname((_Atomic(type)*)ptr, val, model);\ - Lock *l = lock_for_pointer(ptr);\ - lock(l);\ - type tmp = *ptr;\ - *ptr = tmp op val;\ - unlock(l);\ - return tmp;\ -} +#define ATOMIC_RMW(n, lockfree, type, opname, op) \ + type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \ + if (lockfree) \ + return __c11_atomic_fetch_##opname((_Atomic(type) *)ptr, val, model); \ + Lock *l = lock_for_pointer(ptr); \ + lock(l); \ + type tmp = *ptr; \ + *ptr = tmp op val; \ + unlock(l); \ + return tmp; \ + } #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +) OPTIMISED_CASES diff --git a/lib/builtins/atomic_flag_clear.c b/lib/builtins/atomic_flag_clear.c index da912af64312..983e5d7f076b 100644 --- a/lib/builtins/atomic_flag_clear.c +++ b/lib/builtins/atomic_flag_clear.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_clear.c -------------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_clear from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_clear.c -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_clear from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_flag_clear_explicit.c b/lib/builtins/atomic_flag_clear_explicit.c index 1059b787f169..e61c0647684e 100644 --- a/lib/builtins/atomic_flag_clear_explicit.c +++ b/lib/builtins/atomic_flag_clear_explicit.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_clear_explicit.c ----------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_clear_explicit from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_clear_explicit.c --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_clear_explicit from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_flag_test_and_set.c b/lib/builtins/atomic_flag_test_and_set.c index e8811d39ef25..ee22b08b5669 100644 --- a/lib/builtins/atomic_flag_test_and_set.c +++ b/lib/builtins/atomic_flag_test_and_set.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_test_and_set.c ------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_test_and_set from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_test_and_set.c ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_test_and_set from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_flag_test_and_set_explicit.c b/lib/builtins/atomic_flag_test_and_set_explicit.c index 5c8c2df90543..8c9d03994267 100644 --- a/lib/builtins/atomic_flag_test_and_set_explicit.c +++ b/lib/builtins/atomic_flag_test_and_set_explicit.c @@ -1,16 +1,14 @@ -/*===-- atomic_flag_test_and_set_explicit.c ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_flag_test_and_set_explicit from C11's stdatomic.h - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_flag_test_and_set_explicit.c -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_flag_test_and_set_explicit from C11's stdatomic.h +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_signal_fence.c b/lib/builtins/atomic_signal_fence.c index 9ccc2ae60ad8..f4f5169d3008 100644 --- a/lib/builtins/atomic_signal_fence.c +++ b/lib/builtins/atomic_signal_fence.c @@ -1,16 +1,14 @@ -/*===-- atomic_signal_fence.c -----------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_signal_fence from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_signal_fence.c ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_signal_fence from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/atomic_thread_fence.c b/lib/builtins/atomic_thread_fence.c index d22560151bc8..5659ecb0b1f9 100644 --- a/lib/builtins/atomic_thread_fence.c +++ b/lib/builtins/atomic_thread_fence.c @@ -1,16 +1,14 @@ -/*===-- atomic_thread_fence.c -----------------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===------------------------------------------------------------------------=== - * - * This file implements atomic_thread_fence from C11's stdatomic.h. - * - *===------------------------------------------------------------------------=== - */ +//===-- atomic_thread_fence.c ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements atomic_thread_fence from C11's stdatomic.h. +// +//===----------------------------------------------------------------------===// #ifndef __has_include #define __has_include(inc) 0 diff --git a/lib/builtins/bswapdi2.c b/lib/builtins/bswapdi2.c index eb220007bb10..cd049f58e328 100644 --- a/lib/builtins/bswapdi2.c +++ b/lib/builtins/bswapdi2.c @@ -1,16 +1,14 @@ -/* ===-- bswapdi2.c - Implement __bswapdi2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __bswapdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- bswapdi2.c - Implement __bswapdi2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __bswapdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" diff --git a/lib/builtins/bswapsi2.c b/lib/builtins/bswapsi2.c index 5d941e69f7c4..ec566d61a2b0 100644 --- a/lib/builtins/bswapsi2.c +++ b/lib/builtins/bswapsi2.c @@ -1,23 +1,20 @@ -/* ===-- bswapsi2.c - Implement __bswapsi2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __bswapsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- bswapsi2.c - Implement __bswapsi2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __bswapsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" COMPILER_RT_ABI uint32_t __bswapsi2(uint32_t u) { - return ( - (((u)&0xff000000) >> 24) | - (((u)&0x00ff0000) >> 8) | - (((u)&0x0000ff00) << 8) | - (((u)&0x000000ff) << 24)); + return ((((u)&0xff000000) >> 24) | + (((u)&0x00ff0000) >> 8) | + (((u)&0x0000ff00) << 8) | + (((u)&0x000000ff) << 24)); } diff --git a/lib/builtins/clear_cache.c b/lib/builtins/clear_cache.c index 9dcab344ad1b..76dc1968cc7e 100644 --- a/lib/builtins/clear_cache.c +++ b/lib/builtins/clear_cache.c @@ -1,179 +1,164 @@ -/* ===-- clear_cache.c - Implement __clear_cache ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clear_cache.c - Implement __clear_cache ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include <assert.h> #include <stddef.h> #if __APPLE__ - #include <libkern/OSCacheControl.h> +#include <libkern/OSCacheControl.h> #endif #if defined(_WIN32) -/* Forward declare Win32 APIs since the GCC mode driver does not handle the - newer SDKs as well as needed. */ +// Forward declare Win32 APIs since the GCC mode driver does not handle the +// newer SDKs as well as needed. uint32_t FlushInstructionCache(uintptr_t hProcess, void *lpBaseAddress, uintptr_t dwSize); uintptr_t GetCurrentProcess(void); #endif #if defined(__FreeBSD__) && defined(__arm__) - #include <sys/types.h> - #include <machine/sysarch.h> +#include <machine/sysarch.h> +#include <sys/types.h> #endif #if defined(__NetBSD__) && defined(__arm__) - #include <machine/sysarch.h> +#include <machine/sysarch.h> #endif #if defined(__OpenBSD__) && defined(__mips__) - #include <sys/types.h> - #include <machine/sysarch.h> +#include <machine/sysarch.h> +#include <sys/types.h> #endif #if defined(__linux__) && defined(__mips__) - #include <sys/cachectl.h> - #include <sys/syscall.h> - #include <unistd.h> - #if defined(__ANDROID__) && defined(__LP64__) - /* - * clear_mips_cache - Invalidates instruction cache for Mips. - */ - static void clear_mips_cache(const void* Addr, size_t Size) { - __asm__ volatile ( - ".set push\n" - ".set noreorder\n" - ".set noat\n" - "beq %[Size], $zero, 20f\n" /* If size == 0, branch around. */ - "nop\n" - "daddu %[Size], %[Addr], %[Size]\n" /* Calculate end address + 1 */ - "rdhwr $v0, $1\n" /* Get step size for SYNCI. - $1 is $HW_SYNCI_Step */ - "beq $v0, $zero, 20f\n" /* If no caches require - synchronization, branch - around. */ - "nop\n" - "10:\n" - "synci 0(%[Addr])\n" /* Synchronize all caches around - address. */ - "daddu %[Addr], %[Addr], $v0\n" /* Add step size. */ - "sltu $at, %[Addr], %[Size]\n" /* Compare current with end - address. */ - "bne $at, $zero, 10b\n" /* Branch if more to do. */ - "nop\n" - "sync\n" /* Clear memory hazards. */ - "20:\n" - "bal 30f\n" - "nop\n" - "30:\n" - "daddiu $ra, $ra, 12\n" /* $ra has a value of $pc here. - Add offset of 12 to point to the - instruction after the last nop. - */ - "jr.hb $ra\n" /* Return, clearing instruction - hazards. */ - "nop\n" - ".set pop\n" - : [Addr] "+r"(Addr), [Size] "+r"(Size) - :: "at", "ra", "v0", "memory" - ); - } - #endif +#include <sys/cachectl.h> +#include <sys/syscall.h> +#include <unistd.h> +#if defined(__ANDROID__) && defined(__LP64__) +// clear_mips_cache - Invalidates instruction cache for Mips. +static void clear_mips_cache(const void *Addr, size_t Size) { + __asm__ volatile( + ".set push\n" + ".set noreorder\n" + ".set noat\n" + "beq %[Size], $zero, 20f\n" // If size == 0, branch around. + "nop\n" + "daddu %[Size], %[Addr], %[Size]\n" // Calculate end address + 1 + "rdhwr $v0, $1\n" // Get step size for SYNCI. + // $1 is $HW_SYNCI_Step + "beq $v0, $zero, 20f\n" // If no caches require + // synchronization, branch + // around. + "nop\n" + "10:\n" + "synci 0(%[Addr])\n" // Synchronize all caches around + // address. + "daddu %[Addr], %[Addr], $v0\n" // Add step size. + "sltu $at, %[Addr], %[Size]\n" // Compare current with end + // address. + "bne $at, $zero, 10b\n" // Branch if more to do. + "nop\n" + "sync\n" // Clear memory hazards. + "20:\n" + "bal 30f\n" + "nop\n" + "30:\n" + "daddiu $ra, $ra, 12\n" // $ra has a value of $pc here. + // Add offset of 12 to point to the + // instruction after the last nop. + // + "jr.hb $ra\n" // Return, clearing instruction + // hazards. + "nop\n" + ".set pop\n" + : [ Addr ] "+r"(Addr), [ Size ] "+r"(Size)::"at", "ra", "v0", "memory"); +} +#endif #endif -/* - * The compiler generates calls to __clear_cache() when creating - * trampoline functions on the stack for use with nested functions. - * It is expected to invalidate the instruction cache for the - * specified range. - */ +// The compiler generates calls to __clear_cache() when creating +// trampoline functions on the stack for use with nested functions. +// It is expected to invalidate the instruction cache for the +// specified range. void __clear_cache(void *start, void *end) { #if __i386__ || __x86_64__ || defined(_M_IX86) || defined(_M_X64) -/* - * Intel processors have a unified instruction and data cache - * so there is nothing to do - */ +// Intel processors have a unified instruction and data cache +// so there is nothing to do #elif defined(_WIN32) && (defined(__arm__) || defined(__aarch64__)) - FlushInstructionCache(GetCurrentProcess(), start, end - start); + FlushInstructionCache(GetCurrentProcess(), start, end - start); #elif defined(__arm__) && !defined(__APPLE__) - #if defined(__FreeBSD__) || defined(__NetBSD__) - struct arm_sync_icache_args arg; - - arg.addr = (uintptr_t)start; - arg.len = (uintptr_t)end - (uintptr_t)start; - - sysarch(ARM_SYNC_ICACHE, &arg); - #elif defined(__linux__) - /* - * We used to include asm/unistd.h for the __ARM_NR_cacheflush define, but - * it also brought many other unused defines, as well as a dependency on - * kernel headers to be installed. - * - * This value is stable at least since Linux 3.13 and should remain so for - * compatibility reasons, warranting it's re-definition here. - */ - #define __ARM_NR_cacheflush 0x0f0002 - register int start_reg __asm("r0") = (int) (intptr_t) start; - const register int end_reg __asm("r1") = (int) (intptr_t) end; - const register int flags __asm("r2") = 0; - const register int syscall_nr __asm("r7") = __ARM_NR_cacheflush; - __asm __volatile("svc 0x0" - : "=r"(start_reg) - : "r"(syscall_nr), "r"(start_reg), "r"(end_reg), - "r"(flags)); - assert(start_reg == 0 && "Cache flush syscall failed."); - #else - compilerrt_abort(); - #endif +#if defined(__FreeBSD__) || defined(__NetBSD__) + struct arm_sync_icache_args arg; + + arg.addr = (uintptr_t)start; + arg.len = (uintptr_t)end - (uintptr_t)start; + + sysarch(ARM_SYNC_ICACHE, &arg); +#elif defined(__linux__) +// We used to include asm/unistd.h for the __ARM_NR_cacheflush define, but +// it also brought many other unused defines, as well as a dependency on +// kernel headers to be installed. +// +// This value is stable at least since Linux 3.13 and should remain so for +// compatibility reasons, warranting it's re-definition here. +#define __ARM_NR_cacheflush 0x0f0002 + register int start_reg __asm("r0") = (int)(intptr_t)start; + const register int end_reg __asm("r1") = (int)(intptr_t)end; + const register int flags __asm("r2") = 0; + const register int syscall_nr __asm("r7") = __ARM_NR_cacheflush; + __asm __volatile("svc 0x0" + : "=r"(start_reg) + : "r"(syscall_nr), "r"(start_reg), "r"(end_reg), "r"(flags)); + assert(start_reg == 0 && "Cache flush syscall failed."); +#else + compilerrt_abort(); +#endif #elif defined(__linux__) && defined(__mips__) - const uintptr_t start_int = (uintptr_t) start; - const uintptr_t end_int = (uintptr_t) end; - #if defined(__ANDROID__) && defined(__LP64__) - // Call synci implementation for short address range. - const uintptr_t address_range_limit = 256; - if ((end_int - start_int) <= address_range_limit) { - clear_mips_cache(start, (end_int - start_int)); - } else { - syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); - } - #else - syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); - #endif + const uintptr_t start_int = (uintptr_t)start; + const uintptr_t end_int = (uintptr_t)end; +#if defined(__ANDROID__) && defined(__LP64__) + // Call synci implementation for short address range. + const uintptr_t address_range_limit = 256; + if ((end_int - start_int) <= address_range_limit) { + clear_mips_cache(start, (end_int - start_int)); + } else { + syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); + } +#else + syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); +#endif #elif defined(__mips__) && defined(__OpenBSD__) cacheflush(start, (uintptr_t)end - (uintptr_t)start, BCACHE); #elif defined(__aarch64__) && !defined(__APPLE__) - uint64_t xstart = (uint64_t)(uintptr_t) start; - uint64_t xend = (uint64_t)(uintptr_t) end; + uint64_t xstart = (uint64_t)(uintptr_t)start; + uint64_t xend = (uint64_t)(uintptr_t)end; uint64_t addr; // Get Cache Type Info uint64_t ctr_el0; __asm __volatile("mrs %0, ctr_el0" : "=r"(ctr_el0)); - /* - * dc & ic instructions must use 64bit registers so we don't use - * uintptr_t in case this runs in an IPL32 environment. - */ + // dc & ic instructions must use 64bit registers so we don't use + // uintptr_t in case this runs in an IPL32 environment. const size_t dcache_line_size = 4 << ((ctr_el0 >> 16) & 15); for (addr = xstart & ~(dcache_line_size - 1); addr < xend; addr += dcache_line_size) - __asm __volatile("dc cvau, %0" :: "r"(addr)); + __asm __volatile("dc cvau, %0" ::"r"(addr)); __asm __volatile("dsb ish"); const size_t icache_line_size = 4 << ((ctr_el0 >> 0) & 15); for (addr = xstart & ~(icache_line_size - 1); addr < xend; addr += icache_line_size) - __asm __volatile("ic ivau, %0" :: "r"(addr)); + __asm __volatile("ic ivau, %0" ::"r"(addr)); __asm __volatile("isb sy"); -#elif defined (__powerpc64__) +#elif defined(__powerpc64__) const size_t line_size = 32; const size_t len = (uintptr_t)end - (uintptr_t)start; @@ -189,11 +174,11 @@ void __clear_cache(void *start, void *end) { __asm__ volatile("icbi 0, %0" : : "r"(line)); __asm__ volatile("isync"); #else - #if __APPLE__ - /* On Darwin, sys_icache_invalidate() provides this functionality */ - sys_icache_invalidate(start, end-start); - #else - compilerrt_abort(); - #endif +#if __APPLE__ + // On Darwin, sys_icache_invalidate() provides this functionality + sys_icache_invalidate(start, end - start); +#else + compilerrt_abort(); +#endif #endif } diff --git a/lib/builtins/clzdi2.c b/lib/builtins/clzdi2.c index 1819e6be436b..a0bacb2ae39e 100644 --- a/lib/builtins/clzdi2.c +++ b/lib/builtins/clzdi2.c @@ -1,40 +1,35 @@ -/* ===-- clzdi2.c - Implement __clzdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __clzdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzdi2.c - Implement __clzdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __clzdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of leading 0-bits */ +// Returns: the number of leading 0-bits #if !defined(__clang__) && \ - ((defined(__sparc__) && defined(__arch64__)) || \ - defined(__mips64) || \ + ((defined(__sparc__) && defined(__arch64__)) || defined(__mips64) || \ (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) -/* On 64-bit architectures with neither a native clz instruction nor a native - * ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than - * __clzsi2, leading to infinite recursion. */ +// On 64-bit architectures with neither a native clz instruction nor a native +// ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than +// __clzsi2, leading to infinite recursion. #define __builtin_clz(a) __clzsi2(a) extern si_int __clzsi2(si_int); #endif -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__clzdi2(di_int a) -{ - dwords x; - x.all = a; - const si_int f = -(x.s.high == 0); - return __builtin_clz((x.s.high & ~f) | (x.s.low & f)) + - (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __clzdi2(di_int a) { + dwords x; + x.all = a; + const si_int f = -(x.s.high == 0); + return __builtin_clz((x.s.high & ~f) | (x.s.low & f)) + + (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); } diff --git a/lib/builtins/clzsi2.c b/lib/builtins/clzsi2.c index 25b8ed2c4c24..3f9f27f41331 100644 --- a/lib/builtins/clzsi2.c +++ b/lib/builtins/clzsi2.c @@ -1,53 +1,48 @@ -/* ===-- clzsi2.c - Implement __clzsi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __clzsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzsi2.c - Implement __clzsi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __clzsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of leading 0-bits */ +// Returns: the number of leading 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__clzsi2(si_int a) -{ - su_int x = (su_int)a; - si_int t = ((x & 0xFFFF0000) == 0) << 4; /* if (x is small) t = 16 else 0 */ - x >>= 16 - t; /* x = [0 - 0xFFFF] */ - su_int r = t; /* r = [0, 16] */ - /* return r + clz(x) */ - t = ((x & 0xFF00) == 0) << 3; - x >>= 8 - t; /* x = [0 - 0xFF] */ - r += t; /* r = [0, 8, 16, 24] */ - /* return r + clz(x) */ - t = ((x & 0xF0) == 0) << 2; - x >>= 4 - t; /* x = [0 - 0xF] */ - r += t; /* r = [0, 4, 8, 12, 16, 20, 24, 28] */ - /* return r + clz(x) */ - t = ((x & 0xC) == 0) << 1; - x >>= 2 - t; /* x = [0 - 3] */ - r += t; /* r = [0 - 30] and is even */ - /* return r + clz(x) */ -/* switch (x) - * { - * case 0: - * return r + 2; - * case 1: - * return r + 1; - * case 2: - * case 3: - * return r; - * } - */ - return r + ((2 - x) & -((x & 2) == 0)); +COMPILER_RT_ABI si_int __clzsi2(si_int a) { + su_int x = (su_int)a; + si_int t = ((x & 0xFFFF0000) == 0) << 4; // if (x is small) t = 16 else 0 + x >>= 16 - t; // x = [0 - 0xFFFF] + su_int r = t; // r = [0, 16] + // return r + clz(x) + t = ((x & 0xFF00) == 0) << 3; + x >>= 8 - t; // x = [0 - 0xFF] + r += t; // r = [0, 8, 16, 24] + // return r + clz(x) + t = ((x & 0xF0) == 0) << 2; + x >>= 4 - t; // x = [0 - 0xF] + r += t; // r = [0, 4, 8, 12, 16, 20, 24, 28] + // return r + clz(x) + t = ((x & 0xC) == 0) << 1; + x >>= 2 - t; // x = [0 - 3] + r += t; // r = [0 - 30] and is even + // return r + clz(x) + // switch (x) + // { + // case 0: + // return r + 2; + // case 1: + // return r + 1; + // case 2: + // case 3: + // return r; + // } + return r + ((2 - x) & -((x & 2) == 0)); } diff --git a/lib/builtins/clzti2.c b/lib/builtins/clzti2.c index 15a7b3c90004..0c787104caa2 100644 --- a/lib/builtins/clzti2.c +++ b/lib/builtins/clzti2.c @@ -1,33 +1,29 @@ -/* ===-- clzti2.c - Implement __clzti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __clzti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- clzti2.c - Implement __clzti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __clzti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: the number of leading 0-bits */ +// Returns: the number of leading 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__clzti2(ti_int a) -{ - twords x; - x.all = a; - const di_int f = -(x.s.high == 0); - return __builtin_clzll((x.s.high & ~f) | (x.s.low & f)) + - ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __clzti2(ti_int a) { + twords x; + x.all = a; + const di_int f = -(x.s.high == 0); + return __builtin_clzll((x.s.high & ~f) | (x.s.low & f)) + + ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/cmpdi2.c b/lib/builtins/cmpdi2.c index 52634d9c3362..951db85b5062 100644 --- a/lib/builtins/cmpdi2.c +++ b/lib/builtins/cmpdi2.c @@ -1,51 +1,42 @@ -/* ===-- cmpdi2.c - Implement __cmpdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __cmpdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- cmpdi2.c - Implement __cmpdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __cmpdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: if (a < b) returns 0 -* if (a == b) returns 1 -* if (a > b) returns 2 -*/ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__cmpdi2(di_int a, di_int b) -{ - dwords x; - x.all = a; - dwords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __cmpdi2(di_int a, di_int b) { + dwords x; + x.all = a; + dwords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } #ifdef __ARM_EABI__ -/* Returns: if (a < b) returns -1 -* if (a == b) returns 0 -* if (a > b) returns 1 -*/ -COMPILER_RT_ABI si_int -__aeabi_lcmp(di_int a, di_int b) -{ - return __cmpdi2(a, b) - 1; +// Returns: if (a < b) returns -1 +// if (a == b) returns 0 +// if (a > b) returns 1 +COMPILER_RT_ABI si_int __aeabi_lcmp(di_int a, di_int b) { + return __cmpdi2(a, b) - 1; } #endif - diff --git a/lib/builtins/cmpti2.c b/lib/builtins/cmpti2.c index 2c8b56e29a06..7f0ee1b5159d 100644 --- a/lib/builtins/cmpti2.c +++ b/lib/builtins/cmpti2.c @@ -1,42 +1,37 @@ -/* ===-- cmpti2.c - Implement __cmpti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __cmpti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- cmpti2.c - Implement __cmpti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __cmpti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: if (a < b) returns 0 - * if (a == b) returns 1 - * if (a > b) returns 2 - */ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__cmpti2(ti_int a, ti_int b) -{ - twords x; - x.all = a; - twords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __cmpti2(ti_int a, ti_int b) { + twords x; + x.all = a; + twords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/comparedf2.c b/lib/builtins/comparedf2.c index 44e5d2b288a6..58290d87de65 100644 --- a/lib/builtins/comparedf2.c +++ b/lib/builtins/comparedf2.c @@ -1,9 +1,8 @@ //===-- lib/comparedf2.c - Double-precision comparisons -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,79 +39,93 @@ #define DOUBLE_PRECISION #include "fp_lib.h" -enum LE_RESULT { - LE_LESS = -1, - LE_EQUAL = 0, - LE_GREATER = 1, - LE_UNORDERED = 1 -}; +enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 }; + +COMPILER_RT_ABI enum LE_RESULT __ledf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + // If either a or b is NaN, they are unordered. + if (aAbs > infRep || bAbs > infRep) + return LE_UNORDERED; + + // If a and b are both zeros, they are equal. + if ((aAbs | bAbs) == 0) + return LE_EQUAL; -COMPILER_RT_ABI enum LE_RESULT -__ledf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - // If either a or b is NaN, they are unordered. - if (aAbs > infRep || bAbs > infRep) return LE_UNORDERED; - - // If a and b are both zeros, they are equal. - if ((aAbs | bAbs) == 0) return LE_EQUAL; - - // If at least one of a and b is positive, we get the same result comparing - // a and b as signed integers as we would with a floating-point compare. - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } - - // Otherwise, both are negative, so we need to flip the sense of the - // comparison to get the correct result. (This assumes a twos- or ones- - // complement integer representation; if integers are represented in a - // sign-magnitude representation, then this flip is incorrect). - else { - if (aInt > bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } + // If at least one of a and b is positive, we get the same result comparing + // a and b as signed integers as we would with a floating-point compare. + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } + + // Otherwise, both are negative, so we need to flip the sense of the + // comparison to get the correct result. (This assumes a twos- or ones- + // complement integer representation; if integers are represented in a + // sign-magnitude representation, then this flip is incorrect). + else { + if (aInt > bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } } #if defined(__ELF__) // Alias for libgcc compatibility -FNALIAS(__cmpdf2, __ledf2); +COMPILER_RT_ALIAS(__ledf2, __cmpdf2) #endif +COMPILER_RT_ALIAS(__ledf2, __eqdf2) +COMPILER_RT_ALIAS(__ledf2, __ltdf2) +COMPILER_RT_ALIAS(__ledf2, __nedf2) enum GE_RESULT { - GE_LESS = -1, - GE_EQUAL = 0, - GE_GREATER = 1, - GE_UNORDERED = -1 // Note: different from LE_UNORDERED + GE_LESS = -1, + GE_EQUAL = 0, + GE_GREATER = 1, + GE_UNORDERED = -1 // Note: different from LE_UNORDERED }; -COMPILER_RT_ABI enum GE_RESULT -__gedf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - if (aAbs > infRep || bAbs > infRep) return GE_UNORDERED; - if ((aAbs | bAbs) == 0) return GE_EQUAL; - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } else { - if (aInt > bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } +COMPILER_RT_ABI enum GE_RESULT __gedf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + if (aAbs > infRep || bAbs > infRep) + return GE_UNORDERED; + if ((aAbs | bAbs) == 0) + return GE_EQUAL; + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } else { + if (aInt > bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } } +COMPILER_RT_ALIAS(__gedf2, __gtdf2) + COMPILER_RT_ABI int __unorddf2(fp_t a, fp_t b) { const rep_t aAbs = toRep(a) & absMask; @@ -120,34 +133,19 @@ __unorddf2(fp_t a, fp_t b) { return aAbs > infRep || bAbs > infRep; } -// The following are alternative names for the preceding routines. - -COMPILER_RT_ABI enum LE_RESULT -__eqdf2(fp_t a, fp_t b) { - return __ledf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__ltdf2(fp_t a, fp_t b) { - return __ledf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__nedf2(fp_t a, fp_t b) { - return __ledf2(a, b); -} - -COMPILER_RT_ABI enum GE_RESULT -__gtdf2(fp_t a, fp_t b) { - return __gedf2(a, b); -} - #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) { - return __unorddf2(a, b); -} +AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) { return __unorddf2(a, b); } #else -AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unorddf2); +COMPILER_RT_ALIAS(__unorddf2, __aeabi_dcmpun) #endif #endif + +#if defined(_WIN32) && !defined(__MINGW32__) +// The alias mechanism doesn't work on Windows except for MinGW, so emit +// wrapper functions. +int __eqdf2(fp_t a, fp_t b) { return __ledf2(a, b); } +int __ltdf2(fp_t a, fp_t b) { return __ledf2(a, b); } +int __nedf2(fp_t a, fp_t b) { return __ledf2(a, b); } +int __gtdf2(fp_t a, fp_t b) { return __gedf2(a, b); } +#endif diff --git a/lib/builtins/comparesf2.c b/lib/builtins/comparesf2.c index 43cd6a6a7003..1cb99e468c18 100644 --- a/lib/builtins/comparesf2.c +++ b/lib/builtins/comparesf2.c @@ -1,9 +1,8 @@ //===-- lib/comparesf2.c - Single-precision comparisons -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,79 +39,93 @@ #define SINGLE_PRECISION #include "fp_lib.h" -enum LE_RESULT { - LE_LESS = -1, - LE_EQUAL = 0, - LE_GREATER = 1, - LE_UNORDERED = 1 -}; +enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 }; + +COMPILER_RT_ABI enum LE_RESULT __lesf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + // If either a or b is NaN, they are unordered. + if (aAbs > infRep || bAbs > infRep) + return LE_UNORDERED; + + // If a and b are both zeros, they are equal. + if ((aAbs | bAbs) == 0) + return LE_EQUAL; -COMPILER_RT_ABI enum LE_RESULT -__lesf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - // If either a or b is NaN, they are unordered. - if (aAbs > infRep || bAbs > infRep) return LE_UNORDERED; - - // If a and b are both zeros, they are equal. - if ((aAbs | bAbs) == 0) return LE_EQUAL; - - // If at least one of a and b is positive, we get the same result comparing - // a and b as signed integers as we would with a fp_ting-point compare. - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } - - // Otherwise, both are negative, so we need to flip the sense of the - // comparison to get the correct result. (This assumes a twos- or ones- - // complement integer representation; if integers are represented in a - // sign-magnitude representation, then this flip is incorrect). - else { - if (aInt > bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } + // If at least one of a and b is positive, we get the same result comparing + // a and b as signed integers as we would with a fp_ting-point compare. + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } + + // Otherwise, both are negative, so we need to flip the sense of the + // comparison to get the correct result. (This assumes a twos- or ones- + // complement integer representation; if integers are represented in a + // sign-magnitude representation, then this flip is incorrect). + else { + if (aInt > bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } } #if defined(__ELF__) // Alias for libgcc compatibility -FNALIAS(__cmpsf2, __lesf2); +COMPILER_RT_ALIAS(__lesf2, __cmpsf2) #endif +COMPILER_RT_ALIAS(__lesf2, __eqsf2) +COMPILER_RT_ALIAS(__lesf2, __ltsf2) +COMPILER_RT_ALIAS(__lesf2, __nesf2) enum GE_RESULT { - GE_LESS = -1, - GE_EQUAL = 0, - GE_GREATER = 1, - GE_UNORDERED = -1 // Note: different from LE_UNORDERED + GE_LESS = -1, + GE_EQUAL = 0, + GE_GREATER = 1, + GE_UNORDERED = -1 // Note: different from LE_UNORDERED }; -COMPILER_RT_ABI enum GE_RESULT -__gesf2(fp_t a, fp_t b) { - - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - if (aAbs > infRep || bAbs > infRep) return GE_UNORDERED; - if ((aAbs | bAbs) == 0) return GE_EQUAL; - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } else { - if (aInt > bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } +COMPILER_RT_ABI enum GE_RESULT __gesf2(fp_t a, fp_t b) { + + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + if (aAbs > infRep || bAbs > infRep) + return GE_UNORDERED; + if ((aAbs | bAbs) == 0) + return GE_EQUAL; + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } else { + if (aInt > bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } } +COMPILER_RT_ALIAS(__gesf2, __gtsf2) + COMPILER_RT_ABI int __unordsf2(fp_t a, fp_t b) { const rep_t aAbs = toRep(a) & absMask; @@ -120,34 +133,19 @@ __unordsf2(fp_t a, fp_t b) { return aAbs > infRep || bAbs > infRep; } -// The following are alternative names for the preceding routines. - -COMPILER_RT_ABI enum LE_RESULT -__eqsf2(fp_t a, fp_t b) { - return __lesf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__ltsf2(fp_t a, fp_t b) { - return __lesf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT -__nesf2(fp_t a, fp_t b) { - return __lesf2(a, b); -} - -COMPILER_RT_ABI enum GE_RESULT -__gtsf2(fp_t a, fp_t b) { - return __gesf2(a, b); -} - #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { - return __unordsf2(a, b); -} +AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { return __unordsf2(a, b); } #else -AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unordsf2); +COMPILER_RT_ALIAS(__unordsf2, __aeabi_fcmpun) #endif #endif + +#if defined(_WIN32) && !defined(__MINGW32__) +// The alias mechanism doesn't work on Windows except for MinGW, so emit +// wrapper functions. +int __eqsf2(fp_t a, fp_t b) { return __lesf2(a, b); } +int __ltsf2(fp_t a, fp_t b) { return __lesf2(a, b); } +int __nesf2(fp_t a, fp_t b) { return __lesf2(a, b); } +int __gtsf2(fp_t a, fp_t b) { return __gesf2(a, b); } +#endif diff --git a/lib/builtins/comparetf2.c b/lib/builtins/comparetf2.c index c0ad8ed0aecd..2eb34cf37fbc 100644 --- a/lib/builtins/comparetf2.c +++ b/lib/builtins/comparetf2.c @@ -1,9 +1,8 @@ //===-- lib/comparetf2.c - Quad-precision comparisons -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -41,98 +40,95 @@ #include "fp_lib.h" #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) -enum LE_RESULT { - LE_LESS = -1, - LE_EQUAL = 0, - LE_GREATER = 1, - LE_UNORDERED = 1 -}; +enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 }; COMPILER_RT_ABI enum LE_RESULT __letf2(fp_t a, fp_t b) { - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - // If either a or b is NaN, they are unordered. - if (aAbs > infRep || bAbs > infRep) return LE_UNORDERED; - - // If a and b are both zeros, they are equal. - if ((aAbs | bAbs) == 0) return LE_EQUAL; - - // If at least one of a and b is positive, we get the same result comparing - // a and b as signed integers as we would with a floating-point compare. - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } - else { - // Otherwise, both are negative, so we need to flip the sense of the - // comparison to get the correct result. (This assumes a twos- or ones- - // complement integer representation; if integers are represented in a - // sign-magnitude representation, then this flip is incorrect). - if (aInt > bInt) return LE_LESS; - else if (aInt == bInt) return LE_EQUAL; - else return LE_GREATER; - } + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + // If either a or b is NaN, they are unordered. + if (aAbs > infRep || bAbs > infRep) + return LE_UNORDERED; + + // If a and b are both zeros, they are equal. + if ((aAbs | bAbs) == 0) + return LE_EQUAL; + + // If at least one of a and b is positive, we get the same result comparing + // a and b as signed integers as we would with a floating-point compare. + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } else { + // Otherwise, both are negative, so we need to flip the sense of the + // comparison to get the correct result. (This assumes a twos- or ones- + // complement integer representation; if integers are represented in a + // sign-magnitude representation, then this flip is incorrect). + if (aInt > bInt) + return LE_LESS; + else if (aInt == bInt) + return LE_EQUAL; + else + return LE_GREATER; + } } #if defined(__ELF__) // Alias for libgcc compatibility -FNALIAS(__cmptf2, __letf2); +COMPILER_RT_ALIAS(__letf2, __cmptf2) #endif +COMPILER_RT_ALIAS(__letf2, __eqtf2) +COMPILER_RT_ALIAS(__letf2, __lttf2) +COMPILER_RT_ALIAS(__letf2, __netf2) enum GE_RESULT { - GE_LESS = -1, - GE_EQUAL = 0, - GE_GREATER = 1, - GE_UNORDERED = -1 // Note: different from LE_UNORDERED + GE_LESS = -1, + GE_EQUAL = 0, + GE_GREATER = 1, + GE_UNORDERED = -1 // Note: different from LE_UNORDERED }; COMPILER_RT_ABI enum GE_RESULT __getf2(fp_t a, fp_t b) { - const srep_t aInt = toRep(a); - const srep_t bInt = toRep(b); - const rep_t aAbs = aInt & absMask; - const rep_t bAbs = bInt & absMask; - - if (aAbs > infRep || bAbs > infRep) return GE_UNORDERED; - if ((aAbs | bAbs) == 0) return GE_EQUAL; - if ((aInt & bInt) >= 0) { - if (aInt < bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } else { - if (aInt > bInt) return GE_LESS; - else if (aInt == bInt) return GE_EQUAL; - else return GE_GREATER; - } -} - -COMPILER_RT_ABI int __unordtf2(fp_t a, fp_t b) { - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - return aAbs > infRep || bAbs > infRep; -} - -// The following are alternative names for the preceding routines. - -COMPILER_RT_ABI enum LE_RESULT __eqtf2(fp_t a, fp_t b) { - return __letf2(a, b); + const srep_t aInt = toRep(a); + const srep_t bInt = toRep(b); + const rep_t aAbs = aInt & absMask; + const rep_t bAbs = bInt & absMask; + + if (aAbs > infRep || bAbs > infRep) + return GE_UNORDERED; + if ((aAbs | bAbs) == 0) + return GE_EQUAL; + if ((aInt & bInt) >= 0) { + if (aInt < bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } else { + if (aInt > bInt) + return GE_LESS; + else if (aInt == bInt) + return GE_EQUAL; + else + return GE_GREATER; + } } -COMPILER_RT_ABI enum LE_RESULT __lttf2(fp_t a, fp_t b) { - return __letf2(a, b); -} - -COMPILER_RT_ABI enum LE_RESULT __netf2(fp_t a, fp_t b) { - return __letf2(a, b); -} +COMPILER_RT_ALIAS(__getf2, __gttf2) -COMPILER_RT_ABI enum GE_RESULT __gttf2(fp_t a, fp_t b) { - return __getf2(a, b); +COMPILER_RT_ABI int __unordtf2(fp_t a, fp_t b) { + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + return aAbs > infRep || bAbs > infRep; } #endif diff --git a/lib/builtins/cpu_model.c b/lib/builtins/cpu_model.c index fb2b899fc70a..f953aed959e5 100644 --- a/lib/builtins/cpu_model.c +++ b/lib/builtins/cpu_model.c @@ -1,9 +1,8 @@ //===-- cpu_model.c - Support for __cpu_model builtin ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#if (defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64__) || defined(_M_X64)) && \ +#if (defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || \ + defined(_M_X64)) && \ (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) #include <assert.h> @@ -32,8 +31,8 @@ #endif enum VendorSignatures { - SIG_INTEL = 0x756e6547 /* Genu */, - SIG_AMD = 0x68747541 /* Auth */ + SIG_INTEL = 0x756e6547, // Genu + SIG_AMD = 0x68747541, // Auth }; enum ProcessorVendors { @@ -81,6 +80,8 @@ enum ProcessorSubtypes { INTEL_COREI7_CANNONLAKE, INTEL_COREI7_ICELAKE_CLIENT, INTEL_COREI7_ICELAKE_SERVER, + AMDFAM17H_ZNVER2, + INTEL_COREI7_CASCADELAKE, CPU_SUBTYPE_MAX }; @@ -266,10 +267,11 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family, } } -static void -getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, - unsigned Brand_id, unsigned Features, - unsigned *Type, unsigned *Subtype) { +static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, + unsigned Brand_id, + unsigned Features, + unsigned Features2, unsigned *Type, + unsigned *Subtype) { if (Brand_id != 0) return; switch (Family) { @@ -295,7 +297,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, case 0x1e: // Intel(R) Core(TM) i7 CPU 870 @ 2.93GHz. // As found in a Summer 2010 model iMac. case 0x1f: - case 0x2e: // Nehalem EX + case 0x2e: // Nehalem EX *Type = INTEL_COREI7; // "nehalem" *Subtype = INTEL_COREI7_NEHALEM; break; @@ -313,7 +315,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, *Subtype = INTEL_COREI7_SANDYBRIDGE; break; case 0x3a: - case 0x3e: // Ivy Bridge EP + case 0x3e: // Ivy Bridge EP *Type = INTEL_COREI7; // "ivybridge" *Subtype = INTEL_COREI7_IVYBRIDGE; break; @@ -337,10 +339,10 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, break; // Skylake: - case 0x4e: // Skylake mobile - case 0x5e: // Skylake desktop - case 0x8e: // Kaby Lake mobile - case 0x9e: // Kaby Lake desktop + case 0x4e: // Skylake mobile + case 0x5e: // Skylake desktop + case 0x8e: // Kaby Lake mobile + case 0x9e: // Kaby Lake desktop *Type = INTEL_COREI7; // "skylake" *Subtype = INTEL_COREI7_SKYLAKE; break; @@ -348,7 +350,10 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, // Skylake Xeon: case 0x55: *Type = INTEL_COREI7; - *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" + if (Features2 & (1 << (FEATURE_AVX512VNNI - 32))) + *Subtype = INTEL_COREI7_CASCADELAKE; // "cascadelake" + else + *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" break; // Cannonlake: @@ -357,6 +362,20 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, *Subtype = INTEL_COREI7_CANNONLAKE; // "cannonlake" break; + // Icelake: + case 0x7d: + case 0x7e: + *Type = INTEL_COREI7; + *Subtype = INTEL_COREI7_ICELAKE_CLIENT; // "icelake-client" + break; + + // Icelake Xeon: + case 0x6a: + case 0x6c: + *Type = INTEL_COREI7; + *Subtype = INTEL_COREI7_ICELAKE_SERVER; // "icelake-server" + break; + case 0x1c: // Most 45 nm Intel Atom processors case 0x26: // 45 nm Atom Lincroft case 0x27: // 32 nm Atom Medfield @@ -382,6 +401,9 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, case 0x7a: *Type = INTEL_GOLDMONT_PLUS; break; + case 0x86: + *Type = INTEL_TREMONT; + break; case 0x57: *Type = INTEL_KNL; // knl @@ -393,7 +415,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, default: // Unknown family 6 CPU. break; - break; + break; } default: break; // Unknown. @@ -401,8 +423,8 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, - unsigned Features, unsigned *Type, - unsigned *Subtype) { + unsigned Features, unsigned Features2, + unsigned *Type, unsigned *Subtype) { // FIXME: this poorly matches the generated SubtargetFeatureKV table. There // appears to be no way to generate the wide variety of AMD-specific targets // from the information returned from CPUID. @@ -448,7 +470,14 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, break; // "btver2" case 23: *Type = AMDFAM17H; - *Subtype = AMDFAM17H_ZNVER1; + if (Model >= 0x30 && Model <= 0x3f) { + *Subtype = AMDFAM17H_ZNVER2; + break; // "znver2"; 30h-3fh: Zen2 + } + if (Model <= 0x0f) { + *Subtype = AMDFAM17H_ZNVER1; + break; // "znver1"; 00h-0Fh: Zen1 + } break; default: break; // "generic" @@ -462,12 +491,12 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, unsigned Features2 = 0; unsigned EAX, EBX; -#define setFeature(F) \ - do { \ - if (F < 32) \ - Features |= 1U << (F & 0x1f); \ - else if (F < 64) \ - Features2 |= 1U << ((F - 32) & 0x1f); \ +#define setFeature(F) \ + do { \ + if (F < 32) \ + Features |= 1U << (F & 0x1f); \ + else if (F < 64) \ + Features2 |= 1U << ((F - 32) & 0x1f); \ } while (0) if ((EDX >> 15) & 1) @@ -580,24 +609,33 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, #define CONSTRUCTOR_ATTRIBUTE #endif +#ifndef _WIN32 +__attribute__((visibility("hidden"))) +#endif int __cpu_indicator_init(void) CONSTRUCTOR_ATTRIBUTE; +#ifndef _WIN32 +__attribute__((visibility("hidden"))) +#endif struct __processor_model { unsigned int __cpu_vendor; unsigned int __cpu_type; unsigned int __cpu_subtype; unsigned int __cpu_features[1]; } __cpu_model = {0, 0, 0, {0}}; + +#ifndef _WIN32 +__attribute__((visibility("hidden"))) +#endif unsigned int __cpu_features2; -/* A constructor function that is sets __cpu_model and __cpu_features2 with - the right values. This needs to run only once. This constructor is - given the highest priority and it should run before constructors without - the priority set. However, it still runs after ifunc initializers and - needs to be called explicitly there. */ +// A constructor function that is sets __cpu_model and __cpu_features2 with +// the right values. This needs to run only once. This constructor is +// given the highest priority and it should run before constructors without +// the priority set. However, it still runs after ifunc initializers and +// needs to be called explicitly there. -int CONSTRUCTOR_ATTRIBUTE -__cpu_indicator_init(void) { +int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { unsigned EAX, EBX, ECX, EDX; unsigned MaxLeaf = 5; unsigned Vendor; @@ -605,14 +643,14 @@ __cpu_indicator_init(void) { unsigned Features = 0; unsigned Features2 = 0; - /* This function needs to run just once. */ + // This function needs to run just once. if (__cpu_model.__cpu_vendor) return 0; if (!isCpuIdSupported()) return -1; - /* Assume cpuid insn present. Run in level 0 to get vendor id. */ + // Assume cpuid insn present. Run in level 0 to get vendor id. if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) { __cpu_model.__cpu_vendor = VENDOR_OTHER; return -1; @@ -621,20 +659,20 @@ __cpu_indicator_init(void) { detectX86FamilyModel(EAX, &Family, &Model); Brand_id = EBX & 0xff; - /* Find available features. */ + // Find available features. getAvailableFeatures(ECX, EDX, MaxLeaf, &Features, &Features2); __cpu_model.__cpu_features[0] = Features; __cpu_features2 = Features2; if (Vendor == SIG_INTEL) { - /* Get CPU type. */ + // Get CPU type. getIntelProcessorTypeAndSubtype(Family, Model, Brand_id, Features, - &(__cpu_model.__cpu_type), + Features2, &(__cpu_model.__cpu_type), &(__cpu_model.__cpu_subtype)); __cpu_model.__cpu_vendor = VENDOR_INTEL; } else if (Vendor == SIG_AMD) { - /* Get CPU type. */ - getAMDProcessorTypeAndSubtype(Family, Model, Features, + // Get CPU type. + getAMDProcessorTypeAndSubtype(Family, Model, Features, Features2, &(__cpu_model.__cpu_type), &(__cpu_model.__cpu_subtype)); __cpu_model.__cpu_vendor = VENDOR_AMD; diff --git a/lib/builtins/ctzdi2.c b/lib/builtins/ctzdi2.c index ef6d7fea1365..9384aa6055a1 100644 --- a/lib/builtins/ctzdi2.c +++ b/lib/builtins/ctzdi2.c @@ -1,40 +1,35 @@ -/* ===-- ctzdi2.c - Implement __ctzdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ctzdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ctzdi2.c - Implement __ctzdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ctzdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of trailing 0-bits */ +// Returns: the number of trailing 0-bits #if !defined(__clang__) && \ - ((defined(__sparc__) && defined(__arch64__)) || \ - defined(__mips64) || \ + ((defined(__sparc__) && defined(__arch64__)) || defined(__mips64) || \ (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) -/* On 64-bit architectures with neither a native clz instruction nor a native - * ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than - * __ctzsi2, leading to infinite recursion. */ +// On 64-bit architectures with neither a native clz instruction nor a native +// ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than +// __ctzsi2, leading to infinite recursion. #define __builtin_ctz(a) __ctzsi2(a) extern si_int __ctzsi2(si_int); #endif -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__ctzdi2(di_int a) -{ - dwords x; - x.all = a; - const si_int f = -(x.s.low == 0); - return __builtin_ctz((x.s.high & f) | (x.s.low & ~f)) + - (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __ctzdi2(di_int a) { + dwords x; + x.all = a; + const si_int f = -(x.s.low == 0); + return __builtin_ctz((x.s.high & f) | (x.s.low & ~f)) + + (f & ((si_int)(sizeof(si_int) * CHAR_BIT))); } diff --git a/lib/builtins/ctzsi2.c b/lib/builtins/ctzsi2.c index c69486ea445f..09c6863b74e3 100644 --- a/lib/builtins/ctzsi2.c +++ b/lib/builtins/ctzsi2.c @@ -1,57 +1,53 @@ -/* ===-- ctzsi2.c - Implement __ctzsi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ctzsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ctzsi2.c - Implement __ctzsi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ctzsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the number of trailing 0-bits */ +// Returns: the number of trailing 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__ctzsi2(si_int a) -{ - su_int x = (su_int)a; - si_int t = ((x & 0x0000FFFF) == 0) << 4; /* if (x has no small bits) t = 16 else 0 */ - x >>= t; /* x = [0 - 0xFFFF] + higher garbage bits */ - su_int r = t; /* r = [0, 16] */ - /* return r + ctz(x) */ - t = ((x & 0x00FF) == 0) << 3; - x >>= t; /* x = [0 - 0xFF] + higher garbage bits */ - r += t; /* r = [0, 8, 16, 24] */ - /* return r + ctz(x) */ - t = ((x & 0x0F) == 0) << 2; - x >>= t; /* x = [0 - 0xF] + higher garbage bits */ - r += t; /* r = [0, 4, 8, 12, 16, 20, 24, 28] */ - /* return r + ctz(x) */ - t = ((x & 0x3) == 0) << 1; - x >>= t; - x &= 3; /* x = [0 - 3] */ - r += t; /* r = [0 - 30] and is even */ - /* return r + ctz(x) */ +COMPILER_RT_ABI si_int __ctzsi2(si_int a) { + su_int x = (su_int)a; + si_int t = ((x & 0x0000FFFF) == 0) + << 4; // if (x has no small bits) t = 16 else 0 + x >>= t; // x = [0 - 0xFFFF] + higher garbage bits + su_int r = t; // r = [0, 16] + // return r + ctz(x) + t = ((x & 0x00FF) == 0) << 3; + x >>= t; // x = [0 - 0xFF] + higher garbage bits + r += t; // r = [0, 8, 16, 24] + // return r + ctz(x) + t = ((x & 0x0F) == 0) << 2; + x >>= t; // x = [0 - 0xF] + higher garbage bits + r += t; // r = [0, 4, 8, 12, 16, 20, 24, 28] + // return r + ctz(x) + t = ((x & 0x3) == 0) << 1; + x >>= t; + x &= 3; // x = [0 - 3] + r += t; // r = [0 - 30] and is even + // return r + ctz(x) -/* The branch-less return statement below is equivalent - * to the following switch statement: - * switch (x) - * { - * case 0: - * return r + 2; - * case 2: - * return r + 1; - * case 1: - * case 3: - * return r; - * } - */ - return r + ((2 - (x >> 1)) & -((x & 1) == 0)); + // The branch-less return statement below is equivalent + // to the following switch statement: + // switch (x) + // { + // case 0: + // return r + 2; + // case 2: + // return r + 1; + // case 1: + // case 3: + // return r; + // } + return r + ((2 - (x >> 1)) & -((x & 1) == 0)); } diff --git a/lib/builtins/ctzti2.c b/lib/builtins/ctzti2.c index 45de682700cc..2a1312c8437d 100644 --- a/lib/builtins/ctzti2.c +++ b/lib/builtins/ctzti2.c @@ -1,33 +1,29 @@ -/* ===-- ctzti2.c - Implement __ctzti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ctzti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ctzti2.c - Implement __ctzti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ctzti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: the number of trailing 0-bits */ +// Returns: the number of trailing 0-bits -/* Precondition: a != 0 */ +// Precondition: a != 0 -COMPILER_RT_ABI si_int -__ctzti2(ti_int a) -{ - twords x; - x.all = a; - const di_int f = -(x.s.low == 0); - return __builtin_ctzll((x.s.high & f) | (x.s.low & ~f)) + - ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); +COMPILER_RT_ABI si_int __ctzti2(ti_int a) { + twords x; + x.all = a; + const di_int f = -(x.s.low == 0); + return __builtin_ctzll((x.s.high & f) | (x.s.low & ~f)) + + ((si_int)f & ((si_int)(sizeof(di_int) * CHAR_BIT))); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/divdc3.c b/lib/builtins/divdc3.c index 392d6ecacde1..c2cf62874603 100644 --- a/lib/builtins/divdc3.c +++ b/lib/builtins/divdc3.c @@ -1,62 +1,53 @@ -/* ===-- divdc3.c - Implement __divdc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divdc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divdc3.c - Implement __divdc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divdc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Dcomplex -__divdc3(double __a, double __b, double __c, double __d) -{ - int __ilogbw = 0; - double __logbw = __compiler_rt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbn(__c, -__ilogbw); - __d = crt_scalbn(__d, -__ilogbw); +COMPILER_RT_ABI Dcomplex __divdc3(double __a, double __b, double __c, + double __d) { + int __ilogbw = 0; + double __logbw = __compiler_rt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbn(__c, -__ilogbw); + __d = crt_scalbn(__d, -__ilogbw); + } + double __denom = __c * __c + __d * __d; + Dcomplex z; + COMPLEX_REAL(z) = crt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysign(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysign(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysign(crt_isinf(__a) ? 1.0 : 0.0, __a); + __b = crt_copysign(crt_isinf(__b) ? 1.0 : 0.0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0.0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysign(crt_isinf(__c) ? 1.0 : 0.0, __c); + __d = crt_copysign(crt_isinf(__d) ? 1.0 : 0.0, __d); + COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); } - double __denom = __c * __c + __d * __d; - Dcomplex z; - COMPLEX_REAL(z) = crt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysign(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysign(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysign(crt_isinf(__a) ? 1.0 : 0.0, __a); - __b = crt_copysign(crt_isinf(__b) ? 1.0 : 0.0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0.0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysign(crt_isinf(__c) ? 1.0 : 0.0, __c); - __d = crt_copysign(crt_isinf(__d) ? 1.0 : 0.0, __d); - COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } diff --git a/lib/builtins/divdf3.c b/lib/builtins/divdf3.c index 411c82ebb87a..1dea3b534f5a 100644 --- a/lib/builtins/divdf3.c +++ b/lib/builtins/divdf3.c @@ -1,9 +1,8 @@ //===-- lib/divdf3.c - Double-precision division ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,175 +18,193 @@ #define DOUBLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__divdf3(fp_t a, fp_t b) { - - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN / anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything / NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity / infinity = NaN - if (bAbs == infRep) return fromRep(qnanRep); - // infinity / anything else = +/- infinity - else return fromRep(aAbs | quotientSign); - } - - // anything else / infinity = +/- 0 - if (bAbs == infRep) return fromRep(quotientSign); - - if (!aAbs) { - // zero / zero = NaN - if (!bAbs) return fromRep(qnanRep); - // zero / anything else = +/- zero - else return fromRep(quotientSign); - } - // anything else / zero = +/- infinity - if (!bAbs) return fromRep(infRep | quotientSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale -= normalize(&bSignificand); - } - - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - int quotientExponent = aExponent - bExponent + scale; - - // Align the significand of b as a Q31 fixed-point number in the range - // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax - // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This - // is accurate to about 3.5 binary digits. - const uint32_t q31b = bSignificand >> 21; - uint32_t recip32 = UINT32_C(0x7504f333) - q31b; - - // Now refine the reciprocal estimate using a Newton-Raphson iteration: - // - // x1 = x0 * (2 - x0 * b) - // - // This doubles the number of correct binary digits in the approximation - // with each iteration, so after three iterations, we have about 28 binary - // digits of accuracy. - uint32_t correction32; - correction32 = -((uint64_t)recip32 * q31b >> 32); - recip32 = (uint64_t)recip32 * correction32 >> 31; - correction32 = -((uint64_t)recip32 * q31b >> 32); - recip32 = (uint64_t)recip32 * correction32 >> 31; - correction32 = -((uint64_t)recip32 * q31b >> 32); - recip32 = (uint64_t)recip32 * correction32 >> 31; - - // recip32 might have overflowed to exactly zero in the preceding - // computation if the high word of b is exactly 1.0. This would sabotage - // the full-width final stage of the computation that follows, so we adjust - // recip32 downward by one bit. - recip32--; - - // We need to perform one more iteration to get us to 56 binary digits; - // The last iteration needs to happen with extra precision. - const uint32_t q63blo = bSignificand << 11; - uint64_t correction, reciprocal; - correction = -((uint64_t)recip32*q31b + ((uint64_t)recip32*q63blo >> 32)); - uint32_t cHi = correction >> 32; - uint32_t cLo = correction; - reciprocal = (uint64_t)recip32*cHi + ((uint64_t)recip32*cLo >> 32); - - // We already adjusted the 32-bit estimate, now we need to adjust the final - // 64-bit reciprocal estimate downward to ensure that it is strictly smaller - // than the infinitely precise exact reciprocal. Because the computation - // of the Newton-Raphson step is truncating at every step, this adjustment - // is small; most of the work is already done. - reciprocal -= 2; - - // The numerical reciprocal is accurate to within 2^-56, lies in the - // interval [0.5, 1.0), and is strictly smaller than the true reciprocal - // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b - // in Q53 with the following properties: - // - // 1. q < a/b - // 2. q is in the interval [0.5, 2.0) - // 3. the error in q is bounded away from 2^-53 (actually, we have a - // couple of bits to spare, but this is all we need). - - // We need a 64 x 64 multiply high to compute q, which isn't a basic - // operation in C, so we need to be a little bit fussy. - rep_t quotient, quotientLo; - wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); - - // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). - // In either case, we are going to compute a residual of the form - // - // r = a - q*b - // - // We know from the construction of q that r satisfies: - // - // 0 <= r < ulp(q)*b - // - // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we - // already have the correct result. The exact halfway case cannot occur. - // We also take this time to right shift quotient if it falls in the [1,2) - // range and adjust the exponent accordingly. - rep_t residual; - if (quotient < (implicitBit << 1)) { - residual = (aSignificand << 53) - quotient * bSignificand; - quotientExponent--; - } else { - quotient >>= 1; - residual = (aSignificand << 52) - quotient * bSignificand; +COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) { + + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN / anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything / NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity / infinity = NaN + if (bAbs == infRep) + return fromRep(qnanRep); + // infinity / anything else = +/- infinity + else + return fromRep(aAbs | quotientSign); } - const int writtenExponent = quotientExponent + exponentBias; + // anything else / infinity = +/- 0 + if (bAbs == infRep) + return fromRep(quotientSign); - if (writtenExponent >= maxExponent) { - // If we have overflowed the exponent, return infinity. - return fromRep(infRep | quotientSign); - } - - else if (writtenExponent < 1) { - // Flush denormals to zero. In the future, it would be nice to add - // code to round them correctly. + if (!aAbs) { + // zero / zero = NaN + if (!bAbs) + return fromRep(qnanRep); + // zero / anything else = +/- zero + else return fromRep(quotientSign); } - - else { - const bool round = (residual << 1) > bSignificand; - // Clear the implicit bit - rep_t absResult = quotient & significandMask; - // Insert the exponent - absResult |= (rep_t)writtenExponent << significandBits; - // Round - absResult += round; - // Insert the sign and return - const double result = fromRep(absResult | quotientSign); - return result; + // anything else / zero = +/- infinity + if (!bAbs) + return fromRep(infRep | quotientSign); + + // One or both of a or b is denormal. The other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale -= normalize(&bSignificand); + } + + // Set the implicit significand bit. If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything. + aSignificand |= implicitBit; + bSignificand |= implicitBit; + int quotientExponent = aExponent - bExponent + scale; + + // Align the significand of b as a Q31 fixed-point number in the range + // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax + // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This + // is accurate to about 3.5 binary digits. + const uint32_t q31b = bSignificand >> 21; + uint32_t recip32 = UINT32_C(0x7504f333) - q31b; + // 0x7504F333 / 2^32 + 1 = 3/4 + 1/sqrt(2) + + // Now refine the reciprocal estimate using a Newton-Raphson iteration: + // + // x1 = x0 * (2 - x0 * b) + // + // This doubles the number of correct binary digits in the approximation + // with each iteration. + uint32_t correction32; + correction32 = -((uint64_t)recip32 * q31b >> 32); + recip32 = (uint64_t)recip32 * correction32 >> 31; + correction32 = -((uint64_t)recip32 * q31b >> 32); + recip32 = (uint64_t)recip32 * correction32 >> 31; + correction32 = -((uint64_t)recip32 * q31b >> 32); + recip32 = (uint64_t)recip32 * correction32 >> 31; + + // The reciprocal may have overflowed to zero if the upper half of b is + // exactly 1.0. This would sabatoge the full-width final stage of the + // computation that follows, so we adjust the reciprocal down by one bit. + recip32--; + + // We need to perform one more iteration to get us to 56 binary digits. + // The last iteration needs to happen with extra precision. + const uint32_t q63blo = bSignificand << 11; + uint64_t correction, reciprocal; + correction = -((uint64_t)recip32 * q31b + ((uint64_t)recip32 * q63blo >> 32)); + uint32_t cHi = correction >> 32; + uint32_t cLo = correction; + reciprocal = (uint64_t)recip32 * cHi + ((uint64_t)recip32 * cLo >> 32); + + // Adjust the final 64-bit reciprocal estimate downward to ensure that it is + // strictly smaller than the infinitely precise exact reciprocal. Because + // the computation of the Newton-Raphson step is truncating at every step, + // this adjustment is small; most of the work is already done. + reciprocal -= 2; + + // The numerical reciprocal is accurate to within 2^-56, lies in the + // interval [0.5, 1.0), and is strictly smaller than the true reciprocal + // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b + // in Q53 with the following properties: + // + // 1. q < a/b + // 2. q is in the interval [0.5, 2.0) + // 3. The error in q is bounded away from 2^-53 (actually, we have a + // couple of bits to spare, but this is all we need). + + // We need a 64 x 64 multiply high to compute q, which isn't a basic + // operation in C, so we need to be a little bit fussy. + rep_t quotient, quotientLo; + wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); + + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). + // In either case, we are going to compute a residual of the form + // + // r = a - q*b + // + // We know from the construction of q that r satisfies: + // + // 0 <= r < ulp(q)*b + // + // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we + // already have the correct result. The exact halfway case cannot occur. + // We also take this time to right shift quotient if it falls in the [1,2) + // range and adjust the exponent accordingly. + rep_t residual; + if (quotient < (implicitBit << 1)) { + residual = (aSignificand << 53) - quotient * bSignificand; + quotientExponent--; + } else { + quotient >>= 1; + residual = (aSignificand << 52) - quotient * bSignificand; + } + + const int writtenExponent = quotientExponent + exponentBias; + + if (writtenExponent >= maxExponent) { + // If we have overflowed the exponent, return infinity. + return fromRep(infRep | quotientSign); + } + + else if (writtenExponent < 1) { + if (writtenExponent == 0) { + // Check whether the rounded result is normal. + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Round. + absResult += round; + if (absResult & ~significandMask) { + // The rounded result is normal; return it. + return fromRep(absResult | quotientSign); + } } + // Flush denormals to zero. In the future, it would be nice to add + // code to round them correctly. + return fromRep(quotientSign); + } + + else { + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Insert the exponent. + absResult |= (rep_t)writtenExponent << significandBits; + // Round. + absResult += round; + // Insert the sign and return. + const double result = fromRep(absResult | quotientSign); + return result; + } } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) { - return __divdf3(a, b); -} +AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) { return __divdf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divdf3); +COMPILER_RT_ALIAS(__divdf3, __aeabi_ddiv) #endif #endif diff --git a/lib/builtins/divdi3.c b/lib/builtins/divdi3.c index b8eebcb20465..ee08d6557783 100644 --- a/lib/builtins/divdi3.c +++ b/lib/builtins/divdi3.c @@ -1,29 +1,25 @@ -/* ===-- divdi3.c - Implement __divdi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divdi3.c - Implement __divdi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI di_int -__divdi3(di_int a, di_int b) -{ - const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; - di_int s_a = a >> bits_in_dword_m1; /* s_a = a < 0 ? -1 : 0 */ - di_int s_b = b >> bits_in_dword_m1; /* s_b = b < 0 ? -1 : 0 */ - a = (a ^ s_a) - s_a; /* negate if s_a == -1 */ - b = (b ^ s_b) - s_b; /* negate if s_b == -1 */ - s_a ^= s_b; /*sign of quotient */ - return (__udivmoddi4(a, b, (du_int*)0) ^ s_a) - s_a; /* negate if s_a == -1 */ +COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b) { + const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; + di_int s_a = a >> bits_in_dword_m1; // s_a = a < 0 ? -1 : 0 + di_int s_b = b >> bits_in_dword_m1; // s_b = b < 0 ? -1 : 0 + a = (a ^ s_a) - s_a; // negate if s_a == -1 + b = (b ^ s_b) - s_b; // negate if s_b == -1 + s_a ^= s_b; // sign of quotient + return (__udivmoddi4(a, b, (du_int *)0) ^ s_a) - s_a; // negate if s_a == -1 } diff --git a/lib/builtins/divmoddi4.c b/lib/builtins/divmoddi4.c index 0d4df67a63e0..7f333510c003 100644 --- a/lib/builtins/divmoddi4.c +++ b/lib/builtins/divmoddi4.c @@ -1,25 +1,21 @@ -/*===-- divmoddi4.c - Implement __divmoddi4 --------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divmoddi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divmoddi4.c - Implement __divmoddi4 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divmoddi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b, *rem = a % b */ +// Returns: a / b, *rem = a % b -COMPILER_RT_ABI di_int -__divmoddi4(di_int a, di_int b, di_int* rem) -{ - di_int d = __divdi3(a,b); - *rem = a - (d*b); +COMPILER_RT_ABI di_int __divmoddi4(di_int a, di_int b, di_int *rem) { + di_int d = __divdi3(a, b); + *rem = a - (d * b); return d; } diff --git a/lib/builtins/divmodsi4.c b/lib/builtins/divmodsi4.c index dabe2874397c..402eed22fe7a 100644 --- a/lib/builtins/divmodsi4.c +++ b/lib/builtins/divmodsi4.c @@ -1,27 +1,22 @@ -/*===-- divmodsi4.c - Implement __divmodsi4 --------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divmodsi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divmodsi4.c - Implement __divmodsi4 +//--------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divmodsi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b, *rem = a % b */ +// Returns: a / b, *rem = a % b -COMPILER_RT_ABI si_int -__divmodsi4(si_int a, si_int b, si_int* rem) -{ - si_int d = __divsi3(a,b); - *rem = a - (d*b); - return d; +COMPILER_RT_ABI si_int __divmodsi4(si_int a, si_int b, si_int *rem) { + si_int d = __divsi3(a, b); + *rem = a - (d * b); + return d; } - - diff --git a/lib/builtins/divsc3.c b/lib/builtins/divsc3.c index 0d18a256c3de..1a63634dde21 100644 --- a/lib/builtins/divsc3.c +++ b/lib/builtins/divsc3.c @@ -1,63 +1,53 @@ -/*===-- divsc3.c - Implement __divsc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divsc3 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- divsc3.c - Implement __divsc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divsc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Fcomplex -__divsc3(float __a, float __b, float __c, float __d) -{ - int __ilogbw = 0; - float __logbw = - __compiler_rt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbnf(__c, -__ilogbw); - __d = crt_scalbnf(__d, -__ilogbw); +COMPILER_RT_ABI Fcomplex __divsc3(float __a, float __b, float __c, float __d) { + int __ilogbw = 0; + float __logbw = + __compiler_rt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbnf(__c, -__ilogbw); + __d = crt_scalbnf(__d, -__ilogbw); + } + float __denom = __c * __c + __d * __d; + Fcomplex z; + COMPLEX_REAL(z) = crt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysignf(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignf(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); + COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); } - float __denom = __c * __c + __d * __d; - Fcomplex z; - COMPLEX_REAL(z) = crt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysignf(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysignf(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); - COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } diff --git a/lib/builtins/divsf3.c b/lib/builtins/divsf3.c index a74917fd1de5..593f93b45ac2 100644 --- a/lib/builtins/divsf3.c +++ b/lib/builtins/divsf3.c @@ -1,9 +1,8 @@ //===-- lib/divsf3.c - Single-precision division ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,159 +18,177 @@ #define SINGLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__divsf3(fp_t a, fp_t b) { - - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN / anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything / NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity / infinity = NaN - if (bAbs == infRep) return fromRep(qnanRep); - // infinity / anything else = +/- infinity - else return fromRep(aAbs | quotientSign); - } - - // anything else / infinity = +/- 0 - if (bAbs == infRep) return fromRep(quotientSign); - - if (!aAbs) { - // zero / zero = NaN - if (!bAbs) return fromRep(qnanRep); - // zero / anything else = +/- zero - else return fromRep(quotientSign); - } - // anything else / zero = +/- infinity - if (!bAbs) return fromRep(infRep | quotientSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale -= normalize(&bSignificand); - } - - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - int quotientExponent = aExponent - bExponent + scale; - - // Align the significand of b as a Q31 fixed-point number in the range - // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax - // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This - // is accurate to about 3.5 binary digits. - uint32_t q31b = bSignificand << 8; - uint32_t reciprocal = UINT32_C(0x7504f333) - q31b; - - // Now refine the reciprocal estimate using a Newton-Raphson iteration: - // - // x1 = x0 * (2 - x0 * b) - // - // This doubles the number of correct binary digits in the approximation - // with each iteration, so after three iterations, we have about 28 binary - // digits of accuracy. - uint32_t correction; - correction = -((uint64_t)reciprocal * q31b >> 32); - reciprocal = (uint64_t)reciprocal * correction >> 31; - correction = -((uint64_t)reciprocal * q31b >> 32); - reciprocal = (uint64_t)reciprocal * correction >> 31; - correction = -((uint64_t)reciprocal * q31b >> 32); - reciprocal = (uint64_t)reciprocal * correction >> 31; - - // Exhaustive testing shows that the error in reciprocal after three steps - // is in the interval [-0x1.f58108p-31, 0x1.d0e48cp-29], in line with our - // expectations. We bump the reciprocal by a tiny value to force the error - // to be strictly positive (in the range [0x1.4fdfp-37,0x1.287246p-29], to - // be specific). This also causes 1/1 to give a sensible approximation - // instead of zero (due to overflow). - reciprocal -= 2; - - // The numerical reciprocal is accurate to within 2^-28, lies in the - // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller - // than the true reciprocal of b. Multiplying a by this reciprocal thus - // gives a numerical q = a/b in Q24 with the following properties: - // - // 1. q < a/b - // 2. q is in the interval [0x1.000000eep-1, 0x1.fffffffcp0) - // 3. the error in q is at most 2^-24 + 2^-27 -- the 2^24 term comes - // from the fact that we truncate the product, and the 2^27 term - // is the error in the reciprocal of b scaled by the maximum - // possible value of a. As a consequence of this error bound, - // either q or nextafter(q) is the correctly rounded - rep_t quotient = (uint64_t)reciprocal*(aSignificand << 1) >> 32; - - // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). - // In either case, we are going to compute a residual of the form - // - // r = a - q*b - // - // We know from the construction of q that r satisfies: - // - // 0 <= r < ulp(q)*b - // - // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we - // already have the correct result. The exact halfway case cannot occur. - // We also take this time to right shift quotient if it falls in the [1,2) - // range and adjust the exponent accordingly. - rep_t residual; - if (quotient < (implicitBit << 1)) { - residual = (aSignificand << 24) - quotient * bSignificand; - quotientExponent--; - } else { - quotient >>= 1; - residual = (aSignificand << 23) - quotient * bSignificand; +COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) { + + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN / anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything / NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity / infinity = NaN + if (bAbs == infRep) + return fromRep(qnanRep); + // infinity / anything else = +/- infinity + else + return fromRep(aAbs | quotientSign); } - const int writtenExponent = quotientExponent + exponentBias; + // anything else / infinity = +/- 0 + if (bAbs == infRep) + return fromRep(quotientSign); - if (writtenExponent >= maxExponent) { - // If we have overflowed the exponent, return infinity. - return fromRep(infRep | quotientSign); - } - - else if (writtenExponent < 1) { - // Flush denormals to zero. In the future, it would be nice to add - // code to round them correctly. + if (!aAbs) { + // zero / zero = NaN + if (!bAbs) + return fromRep(qnanRep); + // zero / anything else = +/- zero + else return fromRep(quotientSign); } - - else { - const bool round = (residual << 1) > bSignificand; - // Clear the implicit bit - rep_t absResult = quotient & significandMask; - // Insert the exponent - absResult |= (rep_t)writtenExponent << significandBits; - // Round - absResult += round; - // Insert the sign and return + // anything else / zero = +/- infinity + if (!bAbs) + return fromRep(infRep | quotientSign); + + // One or both of a or b is denormal. The other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale -= normalize(&bSignificand); + } + + // Set the implicit significand bit. If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything. + aSignificand |= implicitBit; + bSignificand |= implicitBit; + int quotientExponent = aExponent - bExponent + scale; + // 0x7504F333 / 2^32 + 1 = 3/4 + 1/sqrt(2) + + // Align the significand of b as a Q31 fixed-point number in the range + // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax + // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This + // is accurate to about 3.5 binary digits. + uint32_t q31b = bSignificand << 8; + uint32_t reciprocal = UINT32_C(0x7504f333) - q31b; + + // Now refine the reciprocal estimate using a Newton-Raphson iteration: + // + // x1 = x0 * (2 - x0 * b) + // + // This doubles the number of correct binary digits in the approximation + // with each iteration. + uint32_t correction; + correction = -((uint64_t)reciprocal * q31b >> 32); + reciprocal = (uint64_t)reciprocal * correction >> 31; + correction = -((uint64_t)reciprocal * q31b >> 32); + reciprocal = (uint64_t)reciprocal * correction >> 31; + correction = -((uint64_t)reciprocal * q31b >> 32); + reciprocal = (uint64_t)reciprocal * correction >> 31; + + // Adust the final 32-bit reciprocal estimate downward to ensure that it is + // strictly smaller than the infinitely precise exact reciprocal. Because + // the computation of the Newton-Raphson step is truncating at every step, + // this adjustment is small; most of the work is already done. + reciprocal -= 2; + + // The numerical reciprocal is accurate to within 2^-28, lies in the + // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller + // than the true reciprocal of b. Multiplying a by this reciprocal thus + // gives a numerical q = a/b in Q24 with the following properties: + // + // 1. q < a/b + // 2. q is in the interval [0x1.000000eep-1, 0x1.fffffffcp0) + // 3. The error in q is at most 2^-24 + 2^-27 -- the 2^24 term comes + // from the fact that we truncate the product, and the 2^27 term + // is the error in the reciprocal of b scaled by the maximum + // possible value of a. As a consequence of this error bound, + // either q or nextafter(q) is the correctly rounded. + rep_t quotient = (uint64_t)reciprocal * (aSignificand << 1) >> 32; + + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). + // In either case, we are going to compute a residual of the form + // + // r = a - q*b + // + // We know from the construction of q that r satisfies: + // + // 0 <= r < ulp(q)*b + // + // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we + // already have the correct result. The exact halfway case cannot occur. + // We also take this time to right shift quotient if it falls in the [1,2) + // range and adjust the exponent accordingly. + rep_t residual; + if (quotient < (implicitBit << 1)) { + residual = (aSignificand << 24) - quotient * bSignificand; + quotientExponent--; + } else { + quotient >>= 1; + residual = (aSignificand << 23) - quotient * bSignificand; + } + + const int writtenExponent = quotientExponent + exponentBias; + + if (writtenExponent >= maxExponent) { + // If we have overflowed the exponent, return infinity. + return fromRep(infRep | quotientSign); + } + + else if (writtenExponent < 1) { + if (writtenExponent == 0) { + // Check whether the rounded result is normal. + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Round. + absResult += round; + if (absResult & ~significandMask) { + // The rounded result is normal; return it. return fromRep(absResult | quotientSign); + } } + // Flush denormals to zero. In the future, it would be nice to add + // code to round them correctly. + return fromRep(quotientSign); + } + + else { + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Insert the exponent. + absResult |= (rep_t)writtenExponent << significandBits; + // Round. + absResult += round; + // Insert the sign and return. + return fromRep(absResult | quotientSign); + } } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) { - return __divsf3(a, b); -} +AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) { return __divsf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divsf3); +COMPILER_RT_ALIAS(__divsf3, __aeabi_fdiv) #endif #endif diff --git a/lib/builtins/divsi3.c b/lib/builtins/divsi3.c index 75aea008ddc1..b97e11119f0e 100644 --- a/lib/builtins/divsi3.c +++ b/lib/builtins/divsi3.c @@ -1,39 +1,35 @@ -/* ===-- divsi3.c - Implement __divsi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divsi3.c - Implement __divsi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI si_int -__divsi3(si_int a, si_int b) -{ - const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1; - si_int s_a = a >> bits_in_word_m1; /* s_a = a < 0 ? -1 : 0 */ - si_int s_b = b >> bits_in_word_m1; /* s_b = b < 0 ? -1 : 0 */ - a = (a ^ s_a) - s_a; /* negate if s_a == -1 */ - b = (b ^ s_b) - s_b; /* negate if s_b == -1 */ - s_a ^= s_b; /* sign of quotient */ - /* - * On CPUs without unsigned hardware division support, - * this calls __udivsi3 (notice the cast to su_int). - * On CPUs with unsigned hardware division support, - * this uses the unsigned division instruction. - */ - return ((su_int)a/(su_int)b ^ s_a) - s_a; /* negate if s_a == -1 */ +COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b) { + const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1; + si_int s_a = a >> bits_in_word_m1; // s_a = a < 0 ? -1 : 0 + si_int s_b = b >> bits_in_word_m1; // s_b = b < 0 ? -1 : 0 + a = (a ^ s_a) - s_a; // negate if s_a == -1 + b = (b ^ s_b) - s_b; // negate if s_b == -1 + s_a ^= s_b; // sign of quotient + // + // On CPUs without unsigned hardware division support, + // this calls __udivsi3 (notice the cast to su_int). + // On CPUs with unsigned hardware division support, + // this uses the unsigned division instruction. + // + return ((su_int)a / (su_int)b ^ s_a) - s_a; // negate if s_a == -1 } #if defined(__ARM_EABI__) -AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) COMPILER_RT_ALIAS(__divsi3); +COMPILER_RT_ALIAS(__divsi3, __aeabi_idiv) #endif diff --git a/lib/builtins/divtc3.c b/lib/builtins/divtc3.c index e5ea00d841e3..37c71400e370 100644 --- a/lib/builtins/divtc3.c +++ b/lib/builtins/divtc3.c @@ -1,63 +1,54 @@ -/*===-- divtc3.c - Implement __divtc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divtc3 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- divtc3.c - Implement __divtc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divtc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Lcomplex -__divtc3(long double __a, long double __b, long double __c, long double __d) -{ - int __ilogbw = 0; - long double __logbw = - __compiler_rt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbnl(__c, -__ilogbw); - __d = crt_scalbnl(__d, -__ilogbw); +COMPILER_RT_ABI Lcomplex __divtc3(long double __a, long double __b, + long double __c, long double __d) { + int __ilogbw = 0; + long double __logbw = + __compiler_rt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbnl(__c, -__ilogbw); + __d = crt_scalbnl(__d, -__ilogbw); + } + long double __denom = __c * __c + __d * __d; + Lcomplex z; + COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysignl(crt_isinf(__a) ? 1.0 : 0.0, __a); + __b = crt_copysignl(crt_isinf(__b) ? 1.0 : 0.0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0.0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysignl(crt_isinf(__c) ? 1.0 : 0.0, __c); + __d = crt_copysignl(crt_isinf(__d) ? 1.0 : 0.0, __d); + COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); } - long double __denom = __c * __c + __d * __d; - Lcomplex z; - COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysignl(crt_isinf(__a) ? 1.0 : 0.0, __a); - __b = crt_copysignl(crt_isinf(__b) ? 1.0 : 0.0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0.0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysignl(crt_isinf(__c) ? 1.0 : 0.0, __c); - __d = crt_copysignl(crt_isinf(__d) ? 1.0 : 0.0, __d); - COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } diff --git a/lib/builtins/divtf3.c b/lib/builtins/divtf3.c index e81dab826bdd..6e61d2e31b75 100644 --- a/lib/builtins/divtf3.c +++ b/lib/builtins/divtf3.c @@ -1,9 +1,8 @@ //===-- lib/divtf3.c - Quad-precision division --------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,182 +21,201 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __divtf3(fp_t a, fp_t b) { - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN / anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything / NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity / infinity = NaN - if (bAbs == infRep) return fromRep(qnanRep); - // infinity / anything else = +/- infinity - else return fromRep(aAbs | quotientSign); - } - - // anything else / infinity = +/- 0 - if (bAbs == infRep) return fromRep(quotientSign); - - if (!aAbs) { - // zero / zero = NaN - if (!bAbs) return fromRep(qnanRep); - // zero / anything else = +/- zero - else return fromRep(quotientSign); - } - // anything else / zero = +/- infinity - if (!bAbs) return fromRep(infRep | quotientSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale -= normalize(&bSignificand); - } - - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - int quotientExponent = aExponent - bExponent + scale; - - // Align the significand of b as a Q63 fixed-point number in the range - // [1, 2.0) and get a Q64 approximate reciprocal using a small minimax - // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This - // is accurate to about 3.5 binary digits. - const uint64_t q63b = bSignificand >> 49; - uint64_t recip64 = UINT64_C(0x7504f333F9DE6484) - q63b; - // 0x7504f333F9DE6484 / 2^64 + 1 = 3/4 + 1/sqrt(2) - - // Now refine the reciprocal estimate using a Newton-Raphson iteration: - // - // x1 = x0 * (2 - x0 * b) - // - // This doubles the number of correct binary digits in the approximation - // with each iteration. - uint64_t correction64; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - correction64 = -((rep_t)recip64 * q63b >> 64); - recip64 = (rep_t)recip64 * correction64 >> 63; - - // recip64 might have overflowed to exactly zero in the preceeding - // computation if the high word of b is exactly 1.0. This would sabotage - // the full-width final stage of the computation that follows, so we adjust - // recip64 downward by one bit. - recip64--; - - // We need to perform one more iteration to get us to 112 binary digits; - // The last iteration needs to happen with extra precision. - const uint64_t q127blo = bSignificand << 15; - rep_t correction, reciprocal; - - // NOTE: This operation is equivalent to __multi3, which is not implemented - // in some architechure - rep_t r64q63, r64q127, r64cH, r64cL, dummy; - wideMultiply((rep_t)recip64, (rep_t)q63b, &dummy, &r64q63); - wideMultiply((rep_t)recip64, (rep_t)q127blo, &dummy, &r64q127); - - correction = -(r64q63 + (r64q127 >> 64)); - - uint64_t cHi = correction >> 64; - uint64_t cLo = correction; - - wideMultiply((rep_t)recip64, (rep_t)cHi, &dummy, &r64cH); - wideMultiply((rep_t)recip64, (rep_t)cLo, &dummy, &r64cL); - - reciprocal = r64cH + (r64cL >> 64); - - // We already adjusted the 64-bit estimate, now we need to adjust the final - // 128-bit reciprocal estimate downward to ensure that it is strictly smaller - // than the infinitely precise exact reciprocal. Because the computation - // of the Newton-Raphson step is truncating at every step, this adjustment - // is small; most of the work is already done. - reciprocal -= 2; - - // The numerical reciprocal is accurate to within 2^-112, lies in the - // interval [0.5, 1.0), and is strictly smaller than the true reciprocal - // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b - // in Q127 with the following properties: - // - // 1. q < a/b - // 2. q is in the interval [0.5, 2.0) - // 3. the error in q is bounded away from 2^-113 (actually, we have a - // couple of bits to spare, but this is all we need). - - // We need a 128 x 128 multiply high to compute q, which isn't a basic - // operation in C, so we need to be a little bit fussy. - rep_t quotient, quotientLo; - wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); - - // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). - // In either case, we are going to compute a residual of the form - // - // r = a - q*b - // - // We know from the construction of q that r satisfies: - // - // 0 <= r < ulp(q)*b - // - // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we - // already have the correct result. The exact halfway case cannot occur. - // We also take this time to right shift quotient if it falls in the [1,2) - // range and adjust the exponent accordingly. - rep_t residual; - rep_t qb; - - if (quotient < (implicitBit << 1)) { - wideMultiply(quotient, bSignificand, &dummy, &qb); - residual = (aSignificand << 113) - qb; - quotientExponent--; - } else { - quotient >>= 1; - wideMultiply(quotient, bSignificand, &dummy, &qb); - residual = (aSignificand << 112) - qb; + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN / anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything / NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity / infinity = NaN + if (bAbs == infRep) + return fromRep(qnanRep); + // infinity / anything else = +/- infinity + else + return fromRep(aAbs | quotientSign); } - const int writtenExponent = quotientExponent + exponentBias; + // anything else / infinity = +/- 0 + if (bAbs == infRep) + return fromRep(quotientSign); - if (writtenExponent >= maxExponent) { - // If we have overflowed the exponent, return infinity. - return fromRep(infRep | quotientSign); - } - else if (writtenExponent < 1) { - // Flush denormals to zero. In the future, it would be nice to add - // code to round them correctly. + if (!aAbs) { + // zero / zero = NaN + if (!bAbs) + return fromRep(qnanRep); + // zero / anything else = +/- zero + else return fromRep(quotientSign); } - else { - const bool round = (residual << 1) >= bSignificand; - // Clear the implicit bit - rep_t absResult = quotient & significandMask; - // Insert the exponent - absResult |= (rep_t)writtenExponent << significandBits; - // Round - absResult += round; - // Insert the sign and return - const long double result = fromRep(absResult | quotientSign); - return result; + // anything else / zero = +/- infinity + if (!bAbs) + return fromRep(infRep | quotientSign); + + // One or both of a or b is denormal. The other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale -= normalize(&bSignificand); + } + + // Set the implicit significand bit. If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything. + aSignificand |= implicitBit; + bSignificand |= implicitBit; + int quotientExponent = aExponent - bExponent + scale; + + // Align the significand of b as a Q63 fixed-point number in the range + // [1, 2.0) and get a Q64 approximate reciprocal using a small minimax + // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This + // is accurate to about 3.5 binary digits. + const uint64_t q63b = bSignificand >> 49; + uint64_t recip64 = UINT64_C(0x7504f333F9DE6484) - q63b; + // 0x7504f333F9DE6484 / 2^64 + 1 = 3/4 + 1/sqrt(2) + + // Now refine the reciprocal estimate using a Newton-Raphson iteration: + // + // x1 = x0 * (2 - x0 * b) + // + // This doubles the number of correct binary digits in the approximation + // with each iteration. + uint64_t correction64; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + correction64 = -((rep_t)recip64 * q63b >> 64); + recip64 = (rep_t)recip64 * correction64 >> 63; + + // The reciprocal may have overflowed to zero if the upper half of b is + // exactly 1.0. This would sabatoge the full-width final stage of the + // computation that follows, so we adjust the reciprocal down by one bit. + recip64--; + + // We need to perform one more iteration to get us to 112 binary digits; + // The last iteration needs to happen with extra precision. + const uint64_t q127blo = bSignificand << 15; + rep_t correction, reciprocal; + + // NOTE: This operation is equivalent to __multi3, which is not implemented + // in some architechure + rep_t r64q63, r64q127, r64cH, r64cL, dummy; + wideMultiply((rep_t)recip64, (rep_t)q63b, &dummy, &r64q63); + wideMultiply((rep_t)recip64, (rep_t)q127blo, &dummy, &r64q127); + + correction = -(r64q63 + (r64q127 >> 64)); + + uint64_t cHi = correction >> 64; + uint64_t cLo = correction; + + wideMultiply((rep_t)recip64, (rep_t)cHi, &dummy, &r64cH); + wideMultiply((rep_t)recip64, (rep_t)cLo, &dummy, &r64cL); + + reciprocal = r64cH + (r64cL >> 64); + + // Adjust the final 128-bit reciprocal estimate downward to ensure that it + // is strictly smaller than the infinitely precise exact reciprocal. Because + // the computation of the Newton-Raphson step is truncating at every step, + // this adjustment is small; most of the work is already done. + reciprocal -= 2; + + // The numerical reciprocal is accurate to within 2^-112, lies in the + // interval [0.5, 1.0), and is strictly smaller than the true reciprocal + // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b + // in Q127 with the following properties: + // + // 1. q < a/b + // 2. q is in the interval [0.5, 2.0) + // 3. The error in q is bounded away from 2^-113 (actually, we have a + // couple of bits to spare, but this is all we need). + + // We need a 128 x 128 multiply high to compute q, which isn't a basic + // operation in C, so we need to be a little bit fussy. + rep_t quotient, quotientLo; + wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); + + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). + // In either case, we are going to compute a residual of the form + // + // r = a - q*b + // + // We know from the construction of q that r satisfies: + // + // 0 <= r < ulp(q)*b + // + // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we + // already have the correct result. The exact halfway case cannot occur. + // We also take this time to right shift quotient if it falls in the [1,2) + // range and adjust the exponent accordingly. + rep_t residual; + rep_t qb; + + if (quotient < (implicitBit << 1)) { + wideMultiply(quotient, bSignificand, &dummy, &qb); + residual = (aSignificand << 113) - qb; + quotientExponent--; + } else { + quotient >>= 1; + wideMultiply(quotient, bSignificand, &dummy, &qb); + residual = (aSignificand << 112) - qb; + } + + const int writtenExponent = quotientExponent + exponentBias; + + if (writtenExponent >= maxExponent) { + // If we have overflowed the exponent, return infinity. + return fromRep(infRep | quotientSign); + } else if (writtenExponent < 1) { + if (writtenExponent == 0) { + // Check whether the rounded result is normal. + const bool round = (residual << 1) > bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Round. + absResult += round; + if (absResult & ~significandMask) { + // The rounded result is normal; return it. + return fromRep(absResult | quotientSign); + } } + // Flush denormals to zero. In the future, it would be nice to add + // code to round them correctly. + return fromRep(quotientSign); + } else { + const bool round = (residual << 1) >= bSignificand; + // Clear the implicit bit. + rep_t absResult = quotient & significandMask; + // Insert the exponent. + absResult |= (rep_t)writtenExponent << significandBits; + // Round. + absResult += round; + // Insert the sign and return. + const long double result = fromRep(absResult | quotientSign); + return result; + } } #endif diff --git a/lib/builtins/divti3.c b/lib/builtins/divti3.c index c73eae28fe08..6d007fe34654 100644 --- a/lib/builtins/divti3.c +++ b/lib/builtins/divti3.c @@ -1,33 +1,29 @@ -/* ===-- divti3.c - Implement __divti3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- divti3.c - Implement __divti3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI ti_int -__divti3(ti_int a, ti_int b) -{ - const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; - ti_int s_a = a >> bits_in_tword_m1; /* s_a = a < 0 ? -1 : 0 */ - ti_int s_b = b >> bits_in_tword_m1; /* s_b = b < 0 ? -1 : 0 */ - a = (a ^ s_a) - s_a; /* negate if s_a == -1 */ - b = (b ^ s_b) - s_b; /* negate if s_b == -1 */ - s_a ^= s_b; /* sign of quotient */ - return (__udivmodti4(a, b, (tu_int*)0) ^ s_a) - s_a; /* negate if s_a == -1 */ +COMPILER_RT_ABI ti_int __divti3(ti_int a, ti_int b) { + const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; + ti_int s_a = a >> bits_in_tword_m1; // s_a = a < 0 ? -1 : 0 + ti_int s_b = b >> bits_in_tword_m1; // s_b = b < 0 ? -1 : 0 + a = (a ^ s_a) - s_a; // negate if s_a == -1 + b = (b ^ s_b) - s_b; // negate if s_b == -1 + s_a ^= s_b; // sign of quotient + return (__udivmodti4(a, b, (tu_int *)0) ^ s_a) - s_a; // negate if s_a == -1 } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/divxc3.c b/lib/builtins/divxc3.c index 6f49280e5f61..97ffd2eac211 100644 --- a/lib/builtins/divxc3.c +++ b/lib/builtins/divxc3.c @@ -1,63 +1,55 @@ -/* ===-- divxc3.c - Implement __divxc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __divxc3 for the compiler_rt library. - * - */ +//===-- divxc3.c - Implement __divxc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __divxc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" #include "int_math.h" -/* Returns: the quotient of (a + ib) / (c + id) */ +// Returns: the quotient of (a + ib) / (c + id) -COMPILER_RT_ABI Lcomplex -__divxc3(long double __a, long double __b, long double __c, long double __d) -{ - int __ilogbw = 0; - long double __logbw = crt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); - if (crt_isfinite(__logbw)) - { - __ilogbw = (int)__logbw; - __c = crt_scalbnl(__c, -__ilogbw); - __d = crt_scalbnl(__d, -__ilogbw); +COMPILER_RT_ABI Lcomplex __divxc3(long double __a, long double __b, + long double __c, long double __d) { + int __ilogbw = 0; + long double __logbw = crt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); + if (crt_isfinite(__logbw)) { + __ilogbw = (int)__logbw; + __c = crt_scalbnl(__c, -__ilogbw); + __d = crt_scalbnl(__d, -__ilogbw); + } + long double __denom = __c * __c + __d * __d; + Lcomplex z; + COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = + crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) { + COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; + } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && + crt_isfinite(__d)) { + __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); + } else if (crt_isinf(__logbw) && __logbw > 0 && crt_isfinite(__a) && + crt_isfinite(__b)) { + __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); + COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); } - long double __denom = __c * __c + __d * __d; - Lcomplex z; - COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); - COMPLEX_IMAGINARY(z) = crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) - { - COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; - COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; - } - else if ((crt_isinf(__a) || crt_isinf(__b)) && - crt_isfinite(__c) && crt_isfinite(__d)) - { - __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); - } - else if (crt_isinf(__logbw) && __logbw > 0 && - crt_isfinite(__a) && crt_isfinite(__b)) - { - __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); - COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); - COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); - } - } - return z; + } + return z; } #endif diff --git a/lib/builtins/emutls.c b/lib/builtins/emutls.c index ef95a1c260c1..da58feb7b906 100644 --- a/lib/builtins/emutls.c +++ b/lib/builtins/emutls.c @@ -1,37 +1,35 @@ -/* ===---------- emutls.c - Implements __emutls_get_address ---------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===---------- emutls.c - Implements __emutls_get_address ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include <stdint.h> #include <stdlib.h> #include <string.h> #include "int_lib.h" -#include "int_util.h" #ifdef __BIONIC__ -/* There are 4 pthread key cleanup rounds on Bionic. Delay emutls deallocation - to round 2. We need to delay deallocation because: - - Android versions older than M lack __cxa_thread_atexit_impl, so apps - use a pthread key destructor to call C++ destructors. - - Apps might use __thread/thread_local variables in pthread destructors. - We can't wait until the final two rounds, because jemalloc needs two rounds - after the final malloc/free call to free its thread-specific data (see - https://reviews.llvm.org/D46978#1107507). */ +// There are 4 pthread key cleanup rounds on Bionic. Delay emutls deallocation +// to round 2. We need to delay deallocation because: +// - Android versions older than M lack __cxa_thread_atexit_impl, so apps +// use a pthread key destructor to call C++ destructors. +// - Apps might use __thread/thread_local variables in pthread destructors. +// We can't wait until the final two rounds, because jemalloc needs two rounds +// after the final malloc/free call to free its thread-specific data (see +// https://reviews.llvm.org/D46978#1107507). #define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 1 #else #define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 0 #endif typedef struct emutls_address_array { - uintptr_t skip_destructor_rounds; - uintptr_t size; /* number of elements in the 'data' array */ - void* data[]; + uintptr_t skip_destructor_rounds; + uintptr_t size; // number of elements in the 'data' array + void *data[]; } emutls_address_array; static void emutls_shutdown(emutls_address_array *array); @@ -47,359 +45,339 @@ static bool emutls_key_created = false; typedef unsigned int gcc_word __attribute__((mode(word))); typedef unsigned int gcc_pointer __attribute__((mode(pointer))); -/* Default is not to use posix_memalign, so systems like Android - * can use thread local data without heavier POSIX memory allocators. - */ +// Default is not to use posix_memalign, so systems like Android +// can use thread local data without heavier POSIX memory allocators. #ifndef EMUTLS_USE_POSIX_MEMALIGN #define EMUTLS_USE_POSIX_MEMALIGN 0 #endif static __inline void *emutls_memalign_alloc(size_t align, size_t size) { - void *base; + void *base; #if EMUTLS_USE_POSIX_MEMALIGN - if (posix_memalign(&base, align, size) != 0) - abort(); + if (posix_memalign(&base, align, size) != 0) + abort(); #else - #define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void*)) - char* object; - if ((object = (char*)malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) - abort(); - base = (void*)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) - & ~(uintptr_t)(align - 1)); - - ((void**)base)[-1] = object; +#define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void *)) + char *object; + if ((object = (char *)malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) + abort(); + base = (void *)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) & + ~(uintptr_t)(align - 1)); + + ((void **)base)[-1] = object; #endif - return base; + return base; } static __inline void emutls_memalign_free(void *base) { #if EMUTLS_USE_POSIX_MEMALIGN - free(base); + free(base); #else - /* The mallocated address is in ((void**)base)[-1] */ - free(((void**)base)[-1]); + // The mallocated address is in ((void**)base)[-1] + free(((void **)base)[-1]); #endif } static __inline void emutls_setspecific(emutls_address_array *value) { - pthread_setspecific(emutls_pthread_key, (void*) value); + pthread_setspecific(emutls_pthread_key, (void *)value); } -static __inline emutls_address_array* emutls_getspecific() { - return (emutls_address_array*) pthread_getspecific(emutls_pthread_key); +static __inline emutls_address_array *emutls_getspecific() { + return (emutls_address_array *)pthread_getspecific(emutls_pthread_key); } -static void emutls_key_destructor(void* ptr) { - emutls_address_array *array = (emutls_address_array*)ptr; - if (array->skip_destructor_rounds > 0) { - /* emutls is deallocated using a pthread key destructor. These - * destructors are called in several rounds to accommodate destructor - * functions that (re)initialize key values with pthread_setspecific. - * Delay the emutls deallocation to accommodate other end-of-thread - * cleanup tasks like calling thread_local destructors (e.g. the - * __cxa_thread_atexit fallback in libc++abi). - */ - array->skip_destructor_rounds--; - emutls_setspecific(array); - } else { - emutls_shutdown(array); - free(ptr); - } +static void emutls_key_destructor(void *ptr) { + emutls_address_array *array = (emutls_address_array *)ptr; + if (array->skip_destructor_rounds > 0) { + // emutls is deallocated using a pthread key destructor. These + // destructors are called in several rounds to accommodate destructor + // functions that (re)initialize key values with pthread_setspecific. + // Delay the emutls deallocation to accommodate other end-of-thread + // cleanup tasks like calling thread_local destructors (e.g. the + // __cxa_thread_atexit fallback in libc++abi). + array->skip_destructor_rounds--; + emutls_setspecific(array); + } else { + emutls_shutdown(array); + free(ptr); + } } static __inline void emutls_init(void) { - if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) - abort(); - emutls_key_created = true; + if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) + abort(); + emutls_key_created = true; } static __inline void emutls_init_once(void) { - static pthread_once_t once = PTHREAD_ONCE_INIT; - pthread_once(&once, emutls_init); + static pthread_once_t once = PTHREAD_ONCE_INIT; + pthread_once(&once, emutls_init); } -static __inline void emutls_lock() { - pthread_mutex_lock(&emutls_mutex); -} +static __inline void emutls_lock() { pthread_mutex_lock(&emutls_mutex); } -static __inline void emutls_unlock() { - pthread_mutex_unlock(&emutls_mutex); -} +static __inline void emutls_unlock() { pthread_mutex_unlock(&emutls_mutex); } -#else /* _WIN32 */ +#else // _WIN32 -#include <windows.h> +#include <assert.h> #include <malloc.h> #include <stdio.h> -#include <assert.h> +#include <windows.h> static LPCRITICAL_SECTION emutls_mutex; static DWORD emutls_tls_index = TLS_OUT_OF_INDEXES; typedef uintptr_t gcc_word; -typedef void * gcc_pointer; +typedef void *gcc_pointer; static void win_error(DWORD last_err, const char *hint) { - char *buffer = NULL; - if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_MAX_WIDTH_MASK, - NULL, last_err, 0, (LPSTR)&buffer, 1, NULL)) { - fprintf(stderr, "Windows error: %s\n", buffer); - } else { - fprintf(stderr, "Unkown Windows error: %s\n", hint); - } - LocalFree(buffer); + char *buffer = NULL; + if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_MAX_WIDTH_MASK, + NULL, last_err, 0, (LPSTR)&buffer, 1, NULL)) { + fprintf(stderr, "Windows error: %s\n", buffer); + } else { + fprintf(stderr, "Unkown Windows error: %s\n", hint); + } + LocalFree(buffer); } static __inline void win_abort(DWORD last_err, const char *hint) { - win_error(last_err, hint); - abort(); + win_error(last_err, hint); + abort(); } static __inline void *emutls_memalign_alloc(size_t align, size_t size) { - void *base = _aligned_malloc(size, align); - if (!base) - win_abort(GetLastError(), "_aligned_malloc"); - return base; + void *base = _aligned_malloc(size, align); + if (!base) + win_abort(GetLastError(), "_aligned_malloc"); + return base; } -static __inline void emutls_memalign_free(void *base) { - _aligned_free(base); -} +static __inline void emutls_memalign_free(void *base) { _aligned_free(base); } static void emutls_exit(void) { - if (emutls_mutex) { - DeleteCriticalSection(emutls_mutex); - _aligned_free(emutls_mutex); - emutls_mutex = NULL; - } - if (emutls_tls_index != TLS_OUT_OF_INDEXES) { - emutls_shutdown((emutls_address_array*)TlsGetValue(emutls_tls_index)); - TlsFree(emutls_tls_index); - emutls_tls_index = TLS_OUT_OF_INDEXES; - } + if (emutls_mutex) { + DeleteCriticalSection(emutls_mutex); + _aligned_free(emutls_mutex); + emutls_mutex = NULL; + } + if (emutls_tls_index != TLS_OUT_OF_INDEXES) { + emutls_shutdown((emutls_address_array *)TlsGetValue(emutls_tls_index)); + TlsFree(emutls_tls_index); + emutls_tls_index = TLS_OUT_OF_INDEXES; + } } -#pragma warning (push) -#pragma warning (disable : 4100) +#pragma warning(push) +#pragma warning(disable : 4100) static BOOL CALLBACK emutls_init(PINIT_ONCE p0, PVOID p1, PVOID *p2) { - emutls_mutex = (LPCRITICAL_SECTION)_aligned_malloc(sizeof(CRITICAL_SECTION), 16); - if (!emutls_mutex) { - win_error(GetLastError(), "_aligned_malloc"); - return FALSE; - } - InitializeCriticalSection(emutls_mutex); - - emutls_tls_index = TlsAlloc(); - if (emutls_tls_index == TLS_OUT_OF_INDEXES) { - emutls_exit(); - win_error(GetLastError(), "TlsAlloc"); - return FALSE; - } - atexit(&emutls_exit); - return TRUE; + emutls_mutex = + (LPCRITICAL_SECTION)_aligned_malloc(sizeof(CRITICAL_SECTION), 16); + if (!emutls_mutex) { + win_error(GetLastError(), "_aligned_malloc"); + return FALSE; + } + InitializeCriticalSection(emutls_mutex); + + emutls_tls_index = TlsAlloc(); + if (emutls_tls_index == TLS_OUT_OF_INDEXES) { + emutls_exit(); + win_error(GetLastError(), "TlsAlloc"); + return FALSE; + } + atexit(&emutls_exit); + return TRUE; } static __inline void emutls_init_once(void) { - static INIT_ONCE once; - InitOnceExecuteOnce(&once, emutls_init, NULL, NULL); + static INIT_ONCE once; + InitOnceExecuteOnce(&once, emutls_init, NULL, NULL); } -static __inline void emutls_lock() { - EnterCriticalSection(emutls_mutex); -} +static __inline void emutls_lock() { EnterCriticalSection(emutls_mutex); } -static __inline void emutls_unlock() { - LeaveCriticalSection(emutls_mutex); -} +static __inline void emutls_unlock() { LeaveCriticalSection(emutls_mutex); } static __inline void emutls_setspecific(emutls_address_array *value) { - if (TlsSetValue(emutls_tls_index, (LPVOID) value) == 0) - win_abort(GetLastError(), "TlsSetValue"); + if (TlsSetValue(emutls_tls_index, (LPVOID)value) == 0) + win_abort(GetLastError(), "TlsSetValue"); } -static __inline emutls_address_array* emutls_getspecific() { - LPVOID value = TlsGetValue(emutls_tls_index); - if (value == NULL) { - const DWORD err = GetLastError(); - if (err != ERROR_SUCCESS) - win_abort(err, "TlsGetValue"); - } - return (emutls_address_array*) value; +static __inline emutls_address_array *emutls_getspecific() { + LPVOID value = TlsGetValue(emutls_tls_index); + if (value == NULL) { + const DWORD err = GetLastError(); + if (err != ERROR_SUCCESS) + win_abort(err, "TlsGetValue"); + } + return (emutls_address_array *)value; } -/* Provide atomic load/store functions for emutls_get_index if built with MSVC. - */ +// Provide atomic load/store functions for emutls_get_index if built with MSVC. #if !defined(__ATOMIC_RELEASE) #include <intrin.h> enum { __ATOMIC_ACQUIRE = 2, __ATOMIC_RELEASE = 3 }; static __inline uintptr_t __atomic_load_n(void *ptr, unsigned type) { - assert(type == __ATOMIC_ACQUIRE); - // These return the previous value - but since we do an OR with 0, - // it's equivalent to a plain load. + assert(type == __ATOMIC_ACQUIRE); + // These return the previous value - but since we do an OR with 0, + // it's equivalent to a plain load. #ifdef _WIN64 - return InterlockedOr64(ptr, 0); + return InterlockedOr64(ptr, 0); #else - return InterlockedOr(ptr, 0); + return InterlockedOr(ptr, 0); #endif } static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) { - assert(type == __ATOMIC_RELEASE); - InterlockedExchangePointer((void *volatile *)ptr, (void *)val); + assert(type == __ATOMIC_RELEASE); + InterlockedExchangePointer((void *volatile *)ptr, (void *)val); } -#endif /* __ATOMIC_RELEASE */ +#endif // __ATOMIC_RELEASE -#pragma warning (pop) +#pragma warning(pop) -#endif /* _WIN32 */ +#endif // _WIN32 -static size_t emutls_num_object = 0; /* number of allocated TLS objects */ +static size_t emutls_num_object = 0; // number of allocated TLS objects -/* Free the allocated TLS data - */ +// Free the allocated TLS data static void emutls_shutdown(emutls_address_array *array) { - if (array) { - uintptr_t i; - for (i = 0; i < array->size; ++i) { - if (array->data[i]) - emutls_memalign_free(array->data[i]); - } + if (array) { + uintptr_t i; + for (i = 0; i < array->size; ++i) { + if (array->data[i]) + emutls_memalign_free(array->data[i]); } + } } -/* For every TLS variable xyz, - * there is one __emutls_control variable named __emutls_v.xyz. - * If xyz has non-zero initial value, __emutls_v.xyz's "value" - * will point to __emutls_t.xyz, which has the initial value. - */ +// For every TLS variable xyz, +// there is one __emutls_control variable named __emutls_v.xyz. +// If xyz has non-zero initial value, __emutls_v.xyz's "value" +// will point to __emutls_t.xyz, which has the initial value. typedef struct __emutls_control { - /* Must use gcc_word here, instead of size_t, to match GCC. When - gcc_word is larger than size_t, the upper extra bits are all - zeros. We can use variables of size_t to operate on size and - align. */ - gcc_word size; /* size of the object in bytes */ - gcc_word align; /* alignment of the object in bytes */ - union { - uintptr_t index; /* data[index-1] is the object address */ - void* address; /* object address, when in single thread env */ - } object; - void* value; /* null or non-zero initial value for the object */ + // Must use gcc_word here, instead of size_t, to match GCC. When + // gcc_word is larger than size_t, the upper extra bits are all + // zeros. We can use variables of size_t to operate on size and + // align. + gcc_word size; // size of the object in bytes + gcc_word align; // alignment of the object in bytes + union { + uintptr_t index; // data[index-1] is the object address + void *address; // object address, when in single thread env + } object; + void *value; // null or non-zero initial value for the object } __emutls_control; -/* Emulated TLS objects are always allocated at run-time. */ +// Emulated TLS objects are always allocated at run-time. static __inline void *emutls_allocate_object(__emutls_control *control) { - /* Use standard C types, check with gcc's emutls.o. */ - COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); - COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void*)); - - size_t size = control->size; - size_t align = control->align; - void* base; - if (align < sizeof(void*)) - align = sizeof(void*); - /* Make sure that align is power of 2. */ - if ((align & (align - 1)) != 0) - abort(); - - base = emutls_memalign_alloc(align, size); - if (control->value) - memcpy(base, control->value, size); - else - memset(base, 0, size); - return base; -} + // Use standard C types, check with gcc's emutls.o. + COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); + COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void *)); + + size_t size = control->size; + size_t align = control->align; + void *base; + if (align < sizeof(void *)) + align = sizeof(void *); + // Make sure that align is power of 2. + if ((align & (align - 1)) != 0) + abort(); + base = emutls_memalign_alloc(align, size); + if (control->value) + memcpy(base, control->value, size); + else + memset(base, 0, size); + return base; +} -/* Returns control->object.index; set index if not allocated yet. */ +// Returns control->object.index; set index if not allocated yet. static __inline uintptr_t emutls_get_index(__emutls_control *control) { - uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); + uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); + if (!index) { + emutls_init_once(); + emutls_lock(); + index = control->object.index; if (!index) { - emutls_init_once(); - emutls_lock(); - index = control->object.index; - if (!index) { - index = ++emutls_num_object; - __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); - } - emutls_unlock(); + index = ++emutls_num_object; + __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); } - return index; + emutls_unlock(); + } + return index; } -/* Updates newly allocated thread local emutls_address_array. */ +// Updates newly allocated thread local emutls_address_array. static __inline void emutls_check_array_set_size(emutls_address_array *array, uintptr_t size) { - if (array == NULL) - abort(); - array->size = size; - emutls_setspecific(array); + if (array == NULL) + abort(); + array->size = size; + emutls_setspecific(array); } -/* Returns the new 'data' array size, number of elements, - * which must be no smaller than the given index. - */ +// Returns the new 'data' array size, number of elements, +// which must be no smaller than the given index. static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) { - /* Need to allocate emutls_address_array with extra slots - * to store the header. - * Round up the emutls_address_array size to multiple of 16. - */ - uintptr_t header_words = sizeof(emutls_address_array) / sizeof(void *); - return ((index + header_words + 15) & ~((uintptr_t)15)) - header_words; + // Need to allocate emutls_address_array with extra slots + // to store the header. + // Round up the emutls_address_array size to multiple of 16. + uintptr_t header_words = sizeof(emutls_address_array) / sizeof(void *); + return ((index + header_words + 15) & ~((uintptr_t)15)) - header_words; } -/* Returns the size in bytes required for an emutls_address_array with - * N number of elements for data field. - */ +// Returns the size in bytes required for an emutls_address_array with +// N number of elements for data field. static __inline uintptr_t emutls_asize(uintptr_t N) { - return N * sizeof(void *) + sizeof(emutls_address_array); + return N * sizeof(void *) + sizeof(emutls_address_array); } -/* Returns the thread local emutls_address_array. - * Extends its size if necessary to hold address at index. - */ +// Returns the thread local emutls_address_array. +// Extends its size if necessary to hold address at index. static __inline emutls_address_array * emutls_get_address_array(uintptr_t index) { - emutls_address_array* array = emutls_getspecific(); - if (array == NULL) { - uintptr_t new_size = emutls_new_data_array_size(index); - array = (emutls_address_array*) malloc(emutls_asize(new_size)); - if (array) { - memset(array->data, 0, new_size * sizeof(void*)); - array->skip_destructor_rounds = EMUTLS_SKIP_DESTRUCTOR_ROUNDS; - } - emutls_check_array_set_size(array, new_size); - } else if (index > array->size) { - uintptr_t orig_size = array->size; - uintptr_t new_size = emutls_new_data_array_size(index); - array = (emutls_address_array*) realloc(array, emutls_asize(new_size)); - if (array) - memset(array->data + orig_size, 0, - (new_size - orig_size) * sizeof(void*)); - emutls_check_array_set_size(array, new_size); + emutls_address_array *array = emutls_getspecific(); + if (array == NULL) { + uintptr_t new_size = emutls_new_data_array_size(index); + array = (emutls_address_array *)malloc(emutls_asize(new_size)); + if (array) { + memset(array->data, 0, new_size * sizeof(void *)); + array->skip_destructor_rounds = EMUTLS_SKIP_DESTRUCTOR_ROUNDS; } - return array; + emutls_check_array_set_size(array, new_size); + } else if (index > array->size) { + uintptr_t orig_size = array->size; + uintptr_t new_size = emutls_new_data_array_size(index); + array = (emutls_address_array *)realloc(array, emutls_asize(new_size)); + if (array) + memset(array->data + orig_size, 0, + (new_size - orig_size) * sizeof(void *)); + emutls_check_array_set_size(array, new_size); + } + return array; } -void* __emutls_get_address(__emutls_control* control) { - uintptr_t index = emutls_get_index(control); - emutls_address_array* array = emutls_get_address_array(index--); - if (array->data[index] == NULL) - array->data[index] = emutls_allocate_object(control); - return array->data[index]; +void *__emutls_get_address(__emutls_control *control) { + uintptr_t index = emutls_get_index(control); + emutls_address_array *array = emutls_get_address_array(index--); + if (array->data[index] == NULL) + array->data[index] = emutls_allocate_object(control); + return array->data[index]; } #ifdef __BIONIC__ -/* Called by Bionic on dlclose to delete the emutls pthread key. */ -__attribute__((visibility("hidden"))) -void __emutls_unregister_key(void) { - if (emutls_key_created) { - pthread_key_delete(emutls_pthread_key); - emutls_key_created = false; - } +// Called by Bionic on dlclose to delete the emutls pthread key. +__attribute__((visibility("hidden"))) void __emutls_unregister_key(void) { + if (emutls_key_created) { + pthread_key_delete(emutls_pthread_key); + emutls_key_created = false; + } } #endif diff --git a/lib/builtins/enable_execute_stack.c b/lib/builtins/enable_execute_stack.c index 327d460b4253..e18de4eaebf2 100644 --- a/lib/builtins/enable_execute_stack.c +++ b/lib/builtins/enable_execute_stack.c @@ -1,12 +1,10 @@ -/* ===-- enable_execute_stack.c - Implement __enable_execute_stack ---------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- enable_execute_stack.c - Implement __enable_execute_stack ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -14,10 +12,9 @@ #include <sys/mman.h> #endif -/* #include "config.h" - * FIXME: CMake - include when cmake system is ready. - * Remove #define HAVE_SYSCONF 1 line. - */ +// #include "config.h" +// FIXME: CMake - include when cmake system is ready. +// Remove #define HAVE_SYSCONF 1 line. #define HAVE_SYSCONF 1 #ifdef _WIN32 @@ -26,47 +23,45 @@ #else #ifndef __APPLE__ #include <unistd.h> -#endif /* __APPLE__ */ -#endif /* _WIN32 */ +#endif // __APPLE__ +#endif // _WIN32 #if __LP64__ - #define TRAMPOLINE_SIZE 48 +#define TRAMPOLINE_SIZE 48 #else - #define TRAMPOLINE_SIZE 40 +#define TRAMPOLINE_SIZE 40 #endif -/* - * The compiler generates calls to __enable_execute_stack() when creating - * trampoline functions on the stack for use with nested functions. - * It is expected to mark the page(s) containing the address - * and the next 48 bytes as executable. Since the stack is normally rw- - * that means changing the protection on those page(s) to rwx. - */ +// The compiler generates calls to __enable_execute_stack() when creating +// trampoline functions on the stack for use with nested functions. +// It is expected to mark the page(s) containing the address +// and the next 48 bytes as executable. Since the stack is normally rw- +// that means changing the protection on those page(s) to rwx. -COMPILER_RT_ABI void -__enable_execute_stack(void* addr) -{ +COMPILER_RT_ABI void __enable_execute_stack(void *addr) { #if _WIN32 - MEMORY_BASIC_INFORMATION mbi; - if (!VirtualQuery (addr, &mbi, sizeof(mbi))) - return; /* We should probably assert here because there is no return value */ - VirtualProtect (mbi.BaseAddress, mbi.RegionSize, PAGE_EXECUTE_READWRITE, &mbi.Protect); + MEMORY_BASIC_INFORMATION mbi; + if (!VirtualQuery(addr, &mbi, sizeof(mbi))) + return; // We should probably assert here because there is no return value + VirtualProtect(mbi.BaseAddress, mbi.RegionSize, PAGE_EXECUTE_READWRITE, + &mbi.Protect); #else #if __APPLE__ - /* On Darwin, pagesize is always 4096 bytes */ - const uintptr_t pageSize = 4096; + // On Darwin, pagesize is always 4096 bytes + const uintptr_t pageSize = 4096; #elif !defined(HAVE_SYSCONF) #error "HAVE_SYSCONF not defined! See enable_execute_stack.c" #else - const uintptr_t pageSize = sysconf(_SC_PAGESIZE); -#endif /* __APPLE__ */ + const uintptr_t pageSize = sysconf(_SC_PAGESIZE); +#endif // __APPLE__ - const uintptr_t pageAlignMask = ~(pageSize-1); - uintptr_t p = (uintptr_t)addr; - unsigned char* startPage = (unsigned char*)(p & pageAlignMask); - unsigned char* endPage = (unsigned char*)((p+TRAMPOLINE_SIZE+pageSize) & pageAlignMask); - size_t length = endPage - startPage; - (void) mprotect((void *)startPage, length, PROT_READ | PROT_WRITE | PROT_EXEC); + const uintptr_t pageAlignMask = ~(pageSize - 1); + uintptr_t p = (uintptr_t)addr; + unsigned char *startPage = (unsigned char *)(p & pageAlignMask); + unsigned char *endPage = + (unsigned char *)((p + TRAMPOLINE_SIZE + pageSize) & pageAlignMask); + size_t length = endPage - startPage; + (void)mprotect((void *)startPage, length, PROT_READ | PROT_WRITE | PROT_EXEC); #endif } diff --git a/lib/builtins/eprintf.c b/lib/builtins/eprintf.c index 89f34b154577..89fb0e315b2e 100644 --- a/lib/builtins/eprintf.c +++ b/lib/builtins/eprintf.c @@ -1,35 +1,27 @@ -/* ===---------- eprintf.c - Implements __eprintf --------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ - - +//===---------- eprintf.c - Implements __eprintf --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include <stdio.h> - -/* - * __eprintf() was used in an old version of <assert.h>. - * It can eventually go away, but it is needed when linking - * .o files built with the old <assert.h>. - * - * It should never be exported from a dylib, so it is marked - * visibility hidden. - */ +// __eprintf() was used in an old version of <assert.h>. +// It can eventually go away, but it is needed when linking +// .o files built with the old <assert.h>. +// +// It should never be exported from a dylib, so it is marked +// visibility hidden. #ifndef _WIN32 __attribute__((visibility("hidden"))) #endif COMPILER_RT_ABI void -__eprintf(const char* format, const char* assertion_expression, - const char* line, const char* file) -{ - fprintf(stderr, format, assertion_expression, line, file); - fflush(stderr); - compilerrt_abort(); +__eprintf(const char *format, const char *assertion_expression, + const char *line, const char *file) { + fprintf(stderr, format, assertion_expression, line, file); + fflush(stderr); + compilerrt_abort(); } diff --git a/lib/builtins/extenddftf2.c b/lib/builtins/extenddftf2.c index 86dab8f03a8f..849a39da1915 100644 --- a/lib/builtins/extenddftf2.c +++ b/lib/builtins/extenddftf2.c @@ -1,12 +1,10 @@ //===-- lib/extenddftf2.c - double -> quad conversion -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define QUAD_PRECISION #include "fp_lib.h" @@ -17,7 +15,7 @@ #include "fp_extend_impl.inc" COMPILER_RT_ABI long double __extenddftf2(double a) { - return __extendXfYf2__(a); + return __extendXfYf2__(a); } #endif diff --git a/lib/builtins/extendhfsf2.c b/lib/builtins/extendhfsf2.c index d9c0db84b0ce..7c1a76eb5851 100644 --- a/lib/builtins/extendhfsf2.c +++ b/lib/builtins/extendhfsf2.c @@ -1,12 +1,10 @@ //===-- lib/extendhfsf2.c - half -> single conversion -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define SRC_HALF #define DST_SINGLE @@ -15,19 +13,15 @@ // Use a forwarding definition and noinline to implement a poor man's alias, // as there isn't a good cross-platform way of defining one. COMPILER_RT_ABI NOINLINE float __extendhfsf2(uint16_t a) { - return __extendXfYf2__(a); + return __extendXfYf2__(a); } -COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { - return __extendhfsf2(a); -} +COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { return __extendhfsf2(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_h2f(uint16_t a) { - return __extendhfsf2(a); -} +AEABI_RTABI float __aeabi_h2f(uint16_t a) { return __extendhfsf2(a); } #else -AEABI_RTABI float __aeabi_h2f(uint16_t a) COMPILER_RT_ALIAS(__extendhfsf2); +COMPILER_RT_ALIAS(__extendhfsf2, __aeabi_h2f) #endif #endif diff --git a/lib/builtins/extendsfdf2.c b/lib/builtins/extendsfdf2.c index 3d84529a6c53..8132d57e6bf6 100644 --- a/lib/builtins/extendsfdf2.c +++ b/lib/builtins/extendsfdf2.c @@ -1,27 +1,21 @@ //===-- lib/extendsfdf2.c - single -> double conversion -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define SRC_SINGLE #define DST_DOUBLE #include "fp_extend_impl.inc" -COMPILER_RT_ABI double __extendsfdf2(float a) { - return __extendXfYf2__(a); -} +COMPILER_RT_ABI double __extendsfdf2(float a) { return __extendXfYf2__(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_f2d(float a) { - return __extendsfdf2(a); -} +AEABI_RTABI double __aeabi_f2d(float a) { return __extendsfdf2(a); } #else -AEABI_RTABI double __aeabi_f2d(float a) COMPILER_RT_ALIAS(__extendsfdf2); +COMPILER_RT_ALIAS(__extendsfdf2, __aeabi_f2d) #endif #endif diff --git a/lib/builtins/extendsftf2.c b/lib/builtins/extendsftf2.c index 2eeeba284845..c6368406dde1 100644 --- a/lib/builtins/extendsftf2.c +++ b/lib/builtins/extendsftf2.c @@ -1,12 +1,10 @@ //===-- lib/extendsftf2.c - single -> quad conversion -------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// #define QUAD_PRECISION #include "fp_lib.h" @@ -17,7 +15,7 @@ #include "fp_extend_impl.inc" COMPILER_RT_ABI long double __extendsftf2(float a) { - return __extendXfYf2__(a); + return __extendXfYf2__(a); } #endif diff --git a/lib/builtins/ffsdi2.c b/lib/builtins/ffsdi2.c index a5ac9900ff10..9c1a24260956 100644 --- a/lib/builtins/ffsdi2.c +++ b/lib/builtins/ffsdi2.c @@ -1,33 +1,27 @@ -/* ===-- ffsdi2.c - Implement __ffsdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ffsdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ffsdi2.c - Implement __ffsdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ffsdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the index of the least significant 1-bit in a, or - * the value zero if a is zero. The least significant bit is index one. - */ +// Returns: the index of the least significant 1-bit in a, or +// the value zero if a is zero. The least significant bit is index one. -COMPILER_RT_ABI si_int -__ffsdi2(di_int a) -{ - dwords x; - x.all = a; - if (x.s.low == 0) - { - if (x.s.high == 0) - return 0; - return __builtin_ctz(x.s.high) + (1 + sizeof(si_int) * CHAR_BIT); - } - return __builtin_ctz(x.s.low) + 1; +COMPILER_RT_ABI si_int __ffsdi2(di_int a) { + dwords x; + x.all = a; + if (x.s.low == 0) { + if (x.s.high == 0) + return 0; + return __builtin_ctz(x.s.high) + (1 + sizeof(si_int) * CHAR_BIT); + } + return __builtin_ctz(x.s.low) + 1; } diff --git a/lib/builtins/ffssi2.c b/lib/builtins/ffssi2.c index e5180eff5e08..cba1f72fdc61 100644 --- a/lib/builtins/ffssi2.c +++ b/lib/builtins/ffssi2.c @@ -1,29 +1,23 @@ -/* ===-- ffssi2.c - Implement __ffssi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ffssi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ffssi2.c - Implement __ffssi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ffssi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: the index of the least significant 1-bit in a, or - * the value zero if a is zero. The least significant bit is index one. - */ +// Returns: the index of the least significant 1-bit in a, or +// the value zero if a is zero. The least significant bit is index one. -COMPILER_RT_ABI si_int -__ffssi2(si_int a) -{ - if (a == 0) - { - return 0; - } - return __builtin_ctz(a) + 1; +COMPILER_RT_ABI si_int __ffssi2(si_int a) { + if (a == 0) { + return 0; + } + return __builtin_ctz(a) + 1; } diff --git a/lib/builtins/ffsti2.c b/lib/builtins/ffsti2.c index dcdb3bd7f807..a2d7ce08ada1 100644 --- a/lib/builtins/ffsti2.c +++ b/lib/builtins/ffsti2.c @@ -1,37 +1,31 @@ -/* ===-- ffsti2.c - Implement __ffsti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ffsti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ffsti2.c - Implement __ffsti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ffsti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: the index of the least significant 1-bit in a, or - * the value zero if a is zero. The least significant bit is index one. - */ +// Returns: the index of the least significant 1-bit in a, or +// the value zero if a is zero. The least significant bit is index one. -COMPILER_RT_ABI si_int -__ffsti2(ti_int a) -{ - twords x; - x.all = a; - if (x.s.low == 0) - { - if (x.s.high == 0) - return 0; - return __builtin_ctzll(x.s.high) + (1 + sizeof(di_int) * CHAR_BIT); - } - return __builtin_ctzll(x.s.low) + 1; +COMPILER_RT_ABI si_int __ffsti2(ti_int a) { + twords x; + x.all = a; + if (x.s.low == 0) { + if (x.s.high == 0) + return 0; + return __builtin_ctzll(x.s.high) + (1 + sizeof(di_int) * CHAR_BIT); + } + return __builtin_ctzll(x.s.low) + 1; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixdfdi.c b/lib/builtins/fixdfdi.c index 54e312d3c8f7..2ed5261c51b7 100644 --- a/lib/builtins/fixdfdi.c +++ b/lib/builtins/fixdfdi.c @@ -1,55 +1,44 @@ -/* ===-- fixdfdi.c - Implement __fixdfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixdfdi.c - Implement __fixdfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. COMPILER_RT_ABI du_int __fixunsdfdi(double a); -COMPILER_RT_ABI di_int -__fixdfdi(double a) -{ - if (a < 0.0) { - return -__fixunsdfdi(-a); - } - return __fixunsdfdi(a); +COMPILER_RT_ABI di_int __fixdfdi(double a) { + if (a < 0.0) { + return -__fixunsdfdi(-a); + } + return __fixunsdfdi(a); } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef di_int fixint_t; typedef du_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI di_int -__fixdfdi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI di_int __fixdfdi(fp_t a) { return __fixint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI di_int __aeabi_d2lz(fp_t a) { - return __fixdfdi(a); -} +AEABI_RTABI di_int __aeabi_d2lz(fp_t a) { return __fixdfdi(a); } #else -AEABI_RTABI di_int __aeabi_d2lz(fp_t a) COMPILER_RT_ALIAS(__fixdfdi); +COMPILER_RT_ALIAS(__fixdfdi, __aeabi_d2lz) #endif #endif diff --git a/lib/builtins/fixdfsi.c b/lib/builtins/fixdfsi.c index 5b9588175717..f54649993c7f 100644 --- a/lib/builtins/fixdfsi.c +++ b/lib/builtins/fixdfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixdfsi.c - Implement __fixdfsi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixdfsi.c - Implement __fixdfsi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" @@ -14,17 +12,12 @@ typedef si_int fixint_t; typedef su_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI si_int -__fixdfsi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI si_int __fixdfsi(fp_t a) { return __fixint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI si_int __aeabi_d2iz(fp_t a) { - return __fixdfsi(a); -} +AEABI_RTABI si_int __aeabi_d2iz(fp_t a) { return __fixdfsi(a); } #else -AEABI_RTABI si_int __aeabi_d2iz(fp_t a) COMPILER_RT_ALIAS(__fixdfsi); +COMPILER_RT_ALIAS(__fixdfsi, __aeabi_d2iz) #endif #endif diff --git a/lib/builtins/fixdfti.c b/lib/builtins/fixdfti.c index aaf225e74f86..90ca8959dae1 100644 --- a/lib/builtins/fixdfti.c +++ b/lib/builtins/fixdfti.c @@ -1,12 +1,10 @@ -/* ===-- fixdfti.c - Implement __fixdfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixdfti.c - Implement __fixdfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -18,9 +16,6 @@ typedef ti_int fixint_t; typedef tu_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI ti_int -__fixdfti(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI ti_int __fixdfti(fp_t a) { return __fixint(a); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixsfdi.c b/lib/builtins/fixsfdi.c index 32e87c60889f..615e93d4f8d9 100644 --- a/lib/builtins/fixsfdi.c +++ b/lib/builtins/fixsfdi.c @@ -1,55 +1,44 @@ -/* ===-- fixsfdi.c - Implement __fixsfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixsfdi.c - Implement __fixsfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. COMPILER_RT_ABI du_int __fixunssfdi(float a); -COMPILER_RT_ABI di_int -__fixsfdi(float a) -{ - if (a < 0.0f) { - return -__fixunssfdi(-a); - } - return __fixunssfdi(a); +COMPILER_RT_ABI di_int __fixsfdi(float a) { + if (a < 0.0f) { + return -__fixunssfdi(-a); + } + return __fixunssfdi(a); } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef di_int fixint_t; typedef du_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI di_int -__fixsfdi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI di_int __fixsfdi(fp_t a) { return __fixint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI di_int __aeabi_f2lz(fp_t a) { - return __fixsfdi(a); -} +AEABI_RTABI di_int __aeabi_f2lz(fp_t a) { return __fixsfdi(a); } #else -AEABI_RTABI di_int __aeabi_f2lz(fp_t a) COMPILER_RT_ALIAS(__fixsfdi); +COMPILER_RT_ALIAS(__fixsfdi, __aeabi_f2lz) #endif #endif diff --git a/lib/builtins/fixsfsi.c b/lib/builtins/fixsfsi.c index e94e5f3dcd68..d83d7e722271 100644 --- a/lib/builtins/fixsfsi.c +++ b/lib/builtins/fixsfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixsfsi.c - Implement __fixsfsi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixsfsi.c - Implement __fixsfsi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" @@ -14,17 +12,12 @@ typedef si_int fixint_t; typedef su_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI si_int -__fixsfsi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI si_int __fixsfsi(fp_t a) { return __fixint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI si_int __aeabi_f2iz(fp_t a) { - return __fixsfsi(a); -} +AEABI_RTABI si_int __aeabi_f2iz(fp_t a) { return __fixsfsi(a); } #else -AEABI_RTABI si_int __aeabi_f2iz(fp_t a) COMPILER_RT_ALIAS(__fixsfsi); +COMPILER_RT_ALIAS(__fixsfsi, __aeabi_f2iz) #endif #endif diff --git a/lib/builtins/fixsfti.c b/lib/builtins/fixsfti.c index 3a159b3e18e4..3c01b75e2b23 100644 --- a/lib/builtins/fixsfti.c +++ b/lib/builtins/fixsfti.c @@ -1,12 +1,10 @@ -/* ===-- fixsfti.c - Implement __fixsfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixsfti.c - Implement __fixsfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -18,9 +16,6 @@ typedef ti_int fixint_t; typedef tu_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI ti_int -__fixsfti(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI ti_int __fixsfti(fp_t a) { return __fixint(a); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixtfdi.c b/lib/builtins/fixtfdi.c index bc9dea1f4f81..fe570e6b3755 100644 --- a/lib/builtins/fixtfdi.c +++ b/lib/builtins/fixtfdi.c @@ -1,12 +1,10 @@ -/* ===-- fixtfdi.c - Implement __fixtfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixtfdi.c - Implement __fixtfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -16,8 +14,5 @@ typedef di_int fixint_t; typedef du_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI di_int -__fixtfdi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI di_int __fixtfdi(fp_t a) { return __fixint(a); } #endif diff --git a/lib/builtins/fixtfsi.c b/lib/builtins/fixtfsi.c index feb3de885090..a32bd964caa3 100644 --- a/lib/builtins/fixtfsi.c +++ b/lib/builtins/fixtfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixtfsi.c - Implement __fixtfsi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixtfsi.c - Implement __fixtfsi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -16,8 +14,5 @@ typedef si_int fixint_t; typedef su_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI si_int -__fixtfsi(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI si_int __fixtfsi(fp_t a) { return __fixint(a); } #endif diff --git a/lib/builtins/fixtfti.c b/lib/builtins/fixtfti.c index ee4ada85cb4a..19f84ce38907 100644 --- a/lib/builtins/fixtfti.c +++ b/lib/builtins/fixtfti.c @@ -1,12 +1,10 @@ -/* ===-- fixtfti.c - Implement __fixtfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixtfti.c - Implement __fixtfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -16,8 +14,5 @@ typedef ti_int fixint_t; typedef tu_int fixuint_t; #include "fp_fixint_impl.inc" -COMPILER_RT_ABI ti_int -__fixtfti(fp_t a) { - return __fixint(a); -} +COMPILER_RT_ABI ti_int __fixtfti(fp_t a) { return __fixint(a); } #endif diff --git a/lib/builtins/fixunsdfdi.c b/lib/builtins/fixunsdfdi.c index bfe4dbb25656..d2ba73825cfe 100644 --- a/lib/builtins/fixunsdfdi.c +++ b/lib/builtins/fixunsdfdi.c @@ -1,52 +1,42 @@ -/* ===-- fixunsdfdi.c - Implement __fixunsdfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsdfdi.c - Implement __fixunsdfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ - -COMPILER_RT_ABI du_int -__fixunsdfdi(double a) -{ - if (a <= 0.0) return 0; - su_int high = a / 4294967296.f; /* a / 0x1p32f; */ - su_int low = a - (double)high * 4294967296.f; /* high * 0x1p32f; */ - return ((du_int)high << 32) | low; +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. + +COMPILER_RT_ABI du_int __fixunsdfdi(double a) { + if (a <= 0.0) + return 0; + su_int high = a / 4294967296.f; // a / 0x1p32f; + su_int low = a - (double)high * 4294967296.f; // high * 0x1p32f; + return ((du_int)high << 32) | low; } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef du_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI du_int -__fixunsdfdi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI du_int __fixunsdfdi(fp_t a) { return __fixuint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) { - return __fixunsdfdi(a); -} +AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) { return __fixunsdfdi(a); } #else -AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfdi); +COMPILER_RT_ALIAS(__fixunsdfdi, __aeabi_d2ulz) #endif #endif diff --git a/lib/builtins/fixunsdfsi.c b/lib/builtins/fixunsdfsi.c index 3c5355beae1a..3db2adec09eb 100644 --- a/lib/builtins/fixunsdfsi.c +++ b/lib/builtins/fixunsdfsi.c @@ -1,29 +1,22 @@ -/* ===-- fixunsdfsi.c - Implement __fixunsdfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsdfsi.c - Implement __fixunsdfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define DOUBLE_PRECISION #include "fp_lib.h" typedef su_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI su_int -__fixunsdfsi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI su_int __fixunsdfsi(fp_t a) { return __fixuint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) { - return __fixunsdfsi(a); -} +AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) { return __fixunsdfsi(a); } #else -AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfsi); +COMPILER_RT_ALIAS(__fixunsdfsi, __aeabi_d2uiz) #endif #endif diff --git a/lib/builtins/fixunsdfti.c b/lib/builtins/fixunsdfti.c index f8046a02632b..be497d0e16af 100644 --- a/lib/builtins/fixunsdfti.c +++ b/lib/builtins/fixunsdfti.c @@ -1,12 +1,10 @@ -/* ===-- fixunsdfti.c - Implement __fixunsdfti -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsdfti.c - Implement __fixunsdfti -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" @@ -16,8 +14,5 @@ typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI tu_int -__fixunsdfti(fp_t a) { - return __fixuint(a); -} -#endif /* CRT_HAS_128BIT */ +COMPILER_RT_ABI tu_int __fixunsdfti(fp_t a) { return __fixuint(a); } +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixunssfdi.c b/lib/builtins/fixunssfdi.c index 080a25bb1e99..2b90dafad8c2 100644 --- a/lib/builtins/fixunssfdi.c +++ b/lib/builtins/fixunssfdi.c @@ -1,53 +1,43 @@ -/* ===-- fixunssfdi.c - Implement __fixunssfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunssfdi.c - Implement __fixunssfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; can set the invalid - * flag as a side-effect of computation. - */ - -COMPILER_RT_ABI du_int -__fixunssfdi(float a) -{ - if (a <= 0.0f) return 0; - double da = a; - su_int high = da / 4294967296.f; /* da / 0x1p32f; */ - su_int low = da - (double)high * 4294967296.f; /* high * 0x1p32f; */ - return ((du_int)high << 32) | low; +// Support for systems that have hardware floating-point; can set the invalid +// flag as a side-effect of computation. + +COMPILER_RT_ABI du_int __fixunssfdi(float a) { + if (a <= 0.0f) + return 0; + double da = a; + su_int high = da / 4294967296.f; // da / 0x1p32f; + su_int low = da - (double)high * 4294967296.f; // high * 0x1p32f; + return ((du_int)high << 32) | low; } #else -/* Support for systems that don't have hardware floating-point; there are no - * flags to set, and we don't want to code-gen to an unknown soft-float - * implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. typedef du_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI du_int -__fixunssfdi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI du_int __fixunssfdi(fp_t a) { return __fixuint(a); } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) { - return __fixunssfdi(a); -} +AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) { return __fixunssfdi(a); } #else -AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunssfdi); +COMPILER_RT_ALIAS(__fixunssfdi, __aeabi_f2ulz) #endif #endif diff --git a/lib/builtins/fixunssfsi.c b/lib/builtins/fixunssfsi.c index eca2916a5c88..738c1bb95631 100644 --- a/lib/builtins/fixunssfsi.c +++ b/lib/builtins/fixunssfsi.c @@ -1,33 +1,26 @@ -/* ===-- fixunssfsi.c - Implement __fixunssfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunssfsi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunssfsi.c - Implement __fixunssfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunssfsi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" typedef su_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI su_int -__fixunssfsi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI su_int __fixunssfsi(fp_t a) { return __fixuint(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) { - return __fixunssfsi(a); -} +AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) { return __fixunssfsi(a); } #else -AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunssfsi); +COMPILER_RT_ALIAS(__fixunssfsi, __aeabi_f2uiz) #endif #endif diff --git a/lib/builtins/fixunssfti.c b/lib/builtins/fixunssfti.c index 862d7bd6c7af..5525d77f26f6 100644 --- a/lib/builtins/fixunssfti.c +++ b/lib/builtins/fixunssfti.c @@ -1,16 +1,14 @@ -/* ===-- fixunssfti.c - Implement __fixunssfti -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunssfti for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunssfti.c - Implement __fixunssfti -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunssfti for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #define SINGLE_PRECISION #include "fp_lib.h" @@ -19,8 +17,5 @@ typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI tu_int -__fixunssfti(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI tu_int __fixunssfti(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunstfdi.c b/lib/builtins/fixunstfdi.c index b2995f65834a..a0805e63db82 100644 --- a/lib/builtins/fixunstfdi.c +++ b/lib/builtins/fixunstfdi.c @@ -1,12 +1,10 @@ -/* ===-- fixunstfdi.c - Implement __fixunstfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunstfdi.c - Implement __fixunstfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -15,8 +13,5 @@ typedef du_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI du_int -__fixunstfdi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI du_int __fixunstfdi(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunstfsi.c b/lib/builtins/fixunstfsi.c index b5d3f6a7d38d..3a1320ed3e0a 100644 --- a/lib/builtins/fixunstfsi.c +++ b/lib/builtins/fixunstfsi.c @@ -1,12 +1,10 @@ -/* ===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -15,8 +13,5 @@ typedef su_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI su_int -__fixunstfsi(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI su_int __fixunstfsi(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunstfti.c b/lib/builtins/fixunstfti.c index 22ff9dfc0339..23cd1ab615a7 100644 --- a/lib/builtins/fixunstfti.c +++ b/lib/builtins/fixunstfti.c @@ -1,12 +1,10 @@ -/* ===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunstfsi.c - Implement __fixunstfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #define QUAD_PRECISION #include "fp_lib.h" @@ -15,8 +13,5 @@ typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" -COMPILER_RT_ABI tu_int -__fixunstfti(fp_t a) { - return __fixuint(a); -} +COMPILER_RT_ABI tu_int __fixunstfti(fp_t a) { return __fixuint(a); } #endif diff --git a/lib/builtins/fixunsxfdi.c b/lib/builtins/fixunsxfdi.c index 075304e78dc9..75c4f093794f 100644 --- a/lib/builtins/fixunsxfdi.c +++ b/lib/builtins/fixunsxfdi.c @@ -1,46 +1,39 @@ -/* ===-- fixunsxfdi.c - Implement __fixunsxfdi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunsxfdi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsxfdi.c - Implement __fixunsxfdi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunsxfdi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a unsigned long long, rounding toward zero. - * Negative values all become zero. - */ +// Returns: convert a to a unsigned long long, rounding toward zero. +// Negative values all become zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * du_int is a 64 bit integral type - * value in long double is representable in du_int or is negative - * (no range checking performed) - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes du_int is a 64 bit integral type value in long double is representable +// in du_int or is negative (no range checking performed) -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI du_int -__fixunsxfdi(long double a) -{ - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0 || (fb.u.high.s.low & 0x00008000)) - return 0; - if ((unsigned)e > sizeof(du_int) * CHAR_BIT) - return ~(du_int)0; - return fb.u.low.all >> (63 - e); +COMPILER_RT_ABI du_int __fixunsxfdi(long double a) { + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0 || (fb.u.high.s.low & 0x00008000)) + return 0; + if ((unsigned)e > sizeof(du_int) * CHAR_BIT) + return ~(du_int)0; + return fb.u.low.all >> (63 - e); } #endif diff --git a/lib/builtins/fixunsxfsi.c b/lib/builtins/fixunsxfsi.c index c3c70f743de8..1432d8ba92d2 100644 --- a/lib/builtins/fixunsxfsi.c +++ b/lib/builtins/fixunsxfsi.c @@ -1,45 +1,39 @@ -/* ===-- fixunsxfsi.c - Implement __fixunsxfsi -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunsxfsi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsxfsi.c - Implement __fixunsxfsi -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunsxfsi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a unsigned int, rounding toward zero. - * Negative values all become zero. - */ +// Returns: convert a to a unsigned int, rounding toward zero. +// Negative values all become zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * su_int is a 32 bit integral type - * value in long double is representable in su_int or is negative - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes su_int is a 32 bit integral type value in long double is representable +// in su_int or is negative -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI su_int -__fixunsxfsi(long double a) -{ - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0 || (fb.u.high.s.low & 0x00008000)) - return 0; - if ((unsigned)e > sizeof(su_int) * CHAR_BIT) - return ~(su_int)0; - return fb.u.low.s.high >> (31 - e); +COMPILER_RT_ABI su_int __fixunsxfsi(long double a) { + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0 || (fb.u.high.s.low & 0x00008000)) + return 0; + if ((unsigned)e > sizeof(su_int) * CHAR_BIT) + return ~(su_int)0; + return fb.u.low.s.high >> (31 - e); } -#endif /* !_ARCH_PPC */ +#endif // !_ARCH_PPC diff --git a/lib/builtins/fixunsxfti.c b/lib/builtins/fixunsxfti.c index fb39d00ff5b2..508554e4f8f6 100644 --- a/lib/builtins/fixunsxfti.c +++ b/lib/builtins/fixunsxfti.c @@ -1,50 +1,44 @@ -/* ===-- fixunsxfti.c - Implement __fixunsxfti -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixunsxfti for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixunsxfti.c - Implement __fixunsxfti -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixunsxfti for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a unsigned long long, rounding toward zero. - * Negative values all become zero. - */ +// Returns: convert a to a unsigned long long, rounding toward zero. +// Negative values all become zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * tu_int is a 128 bit integral type - * value in long double is representable in tu_int or is negative - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes tu_int is a 128 bit integral type value in long double is representable +// in tu_int or is negative -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI tu_int -__fixunsxfti(long double a) -{ - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0 || (fb.u.high.s.low & 0x00008000)) - return 0; - if ((unsigned)e > sizeof(tu_int) * CHAR_BIT) - return ~(tu_int)0; - tu_int r = fb.u.low.all; - if (e > 63) - r <<= (e - 63); - else - r >>= (63 - e); - return r; +COMPILER_RT_ABI tu_int __fixunsxfti(long double a) { + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0 || (fb.u.high.s.low & 0x00008000)) + return 0; + if ((unsigned)e > sizeof(tu_int) * CHAR_BIT) + return ~(tu_int)0; + tu_int r = fb.u.low.all; + if (e > 63) + r <<= (e - 63); + else + r >>= (63 - e); + return r; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/fixxfdi.c b/lib/builtins/fixxfdi.c index 011787f9e4b4..4783c0101740 100644 --- a/lib/builtins/fixxfdi.c +++ b/lib/builtins/fixxfdi.c @@ -1,48 +1,43 @@ -/* ===-- fixxfdi.c - Implement __fixxfdi -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixxfdi for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixxfdi.c - Implement __fixxfdi -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixxfdi for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a signed long long, rounding toward zero. */ +// Returns: convert a to a signed long long, rounding toward zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * di_int is a 64 bit integral type - * value in long double is representable in di_int (no range checking performed) - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes di_int is a 64 bit integral type value in long double is representable +// in di_int (no range checking performed) -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI di_int -__fixxfdi(long double a) -{ - const di_int di_max = (di_int)((~(du_int)0) / 2); - const di_int di_min = -di_max - 1; - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0) - return 0; - if ((unsigned)e >= sizeof(di_int) * CHAR_BIT) - return a > 0 ? di_max : di_min; - di_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); - di_int r = fb.u.low.all; - r = (du_int)r >> (63 - e); - return (r ^ s) - s; +COMPILER_RT_ABI di_int __fixxfdi(long double a) { + const di_int di_max = (di_int)((~(du_int)0) / 2); + const di_int di_min = -di_max - 1; + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0) + return 0; + if ((unsigned)e >= sizeof(di_int) * CHAR_BIT) + return a > 0 ? di_max : di_min; + di_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); + di_int r = fb.u.low.all; + r = (du_int)r >> (63 - e); + return (r ^ s) - s; } -#endif /* !_ARCH_PPC */ +#endif // !_ARCH_PPC diff --git a/lib/builtins/fixxfti.c b/lib/builtins/fixxfti.c index 968a4f0d5eea..90e03116e7ca 100644 --- a/lib/builtins/fixxfti.c +++ b/lib/builtins/fixxfti.c @@ -1,51 +1,46 @@ -/* ===-- fixxfti.c - Implement __fixxfti -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __fixxfti for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- fixxfti.c - Implement __fixxfti -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __fixxfti for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a signed long long, rounding toward zero. */ +// Returns: convert a to a signed long long, rounding toward zero. -/* Assumption: long double is an intel 80 bit floating point type padded with 6 bytes - * ti_int is a 128 bit integral type - * value in long double is representable in ti_int - */ +// Assumption: long double is an intel 80 bit floating point type padded with 6 +// bytes ti_int is a 128 bit integral type value in long double is representable +// in ti_int -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI ti_int -__fixxfti(long double a) -{ - const ti_int ti_max = (ti_int)((~(tu_int)0) / 2); - const ti_int ti_min = -ti_max - 1; - long_double_bits fb; - fb.f = a; - int e = (fb.u.high.s.low & 0x00007FFF) - 16383; - if (e < 0) - return 0; - ti_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); - ti_int r = fb.u.low.all; - if ((unsigned)e >= sizeof(ti_int) * CHAR_BIT) - return a > 0 ? ti_max : ti_min; - if (e > 63) - r <<= (e - 63); - else - r >>= (63 - e); - return (r ^ s) - s; +COMPILER_RT_ABI ti_int __fixxfti(long double a) { + const ti_int ti_max = (ti_int)((~(tu_int)0) / 2); + const ti_int ti_min = -ti_max - 1; + long_double_bits fb; + fb.f = a; + int e = (fb.u.high.s.low & 0x00007FFF) - 16383; + if (e < 0) + return 0; + ti_int s = -(si_int)((fb.u.high.s.low & 0x00008000) >> 15); + ti_int r = fb.u.low.all; + if ((unsigned)e >= sizeof(ti_int) * CHAR_BIT) + return a > 0 ? ti_max : ti_min; + if (e > 63) + r <<= (e - 63); + else + r >>= (63 - e); + return (r ^ s) - s; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatdidf.c b/lib/builtins/floatdidf.c index 36b856e078d4..8f887314b9e1 100644 --- a/lib/builtins/floatdidf.c +++ b/lib/builtins/floatdidf.c @@ -1,115 +1,103 @@ -/*===-- floatdidf.c - Implement __floatdidf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * This file implements __floatdidf for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- floatdidf.c - Implement __floatdidf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatdidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: convert a to a double, rounding toward even. */ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * di_int is a 64 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// di_int is a 64 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; we'll set the inexact flag - * as a side-effect of this computation. - */ +// Support for systems that have hardware floating-point; we'll set the inexact +// flag as a side-effect of this computation. -COMPILER_RT_ABI double -__floatdidf(di_int a) -{ - static const double twop52 = 4503599627370496.0; // 0x1.0p52 - static const double twop32 = 4294967296.0; // 0x1.0p32 +COMPILER_RT_ABI double __floatdidf(di_int a) { + static const double twop52 = 4503599627370496.0; // 0x1.0p52 + static const double twop32 = 4294967296.0; // 0x1.0p32 - union { int64_t x; double d; } low = { .d = twop52 }; + union { + int64_t x; + double d; + } low = {.d = twop52}; - const double high = (int32_t)(a >> 32) * twop32; - low.x |= a & INT64_C(0x00000000ffffffff); + const double high = (int32_t)(a >> 32) * twop32; + low.x |= a & INT64_C(0x00000000ffffffff); - const double result = (high - twop52) + low.d; - return result; + const double result = (high - twop52) + low.d; + return result; } #else -/* Support for systems that don't have hardware floating-point; there are no flags to - * set, and we don't want to code-gen to an unknown soft-float implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. -COMPILER_RT_ABI double -__floatdidf(di_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(di_int) * CHAR_BIT; - const di_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = ((du_int)a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((du_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floatdidf(di_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(di_int) * CHAR_BIT; + const di_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = ((du_int)a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((du_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((su_int)s & 0x80000000) | /* sign */ - ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((su_int)s & 0x80000000) | // sign + ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_l2d(di_int a) { - return __floatdidf(a); -} +AEABI_RTABI double __aeabi_l2d(di_int a) { return __floatdidf(a); } #else -AEABI_RTABI double __aeabi_l2d(di_int a) COMPILER_RT_ALIAS(__floatdidf); +COMPILER_RT_ALIAS(__floatdidf, __aeabi_l2d) #endif #endif diff --git a/lib/builtins/floatdisf.c b/lib/builtins/floatdisf.c index a2f09eb2ed2c..cd9e0a3b78a5 100644 --- a/lib/builtins/floatdisf.c +++ b/lib/builtins/floatdisf.c @@ -1,88 +1,75 @@ -/*===-- floatdisf.c - Implement __floatdisf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - * This file implements __floatdisf for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- floatdisf.c - Implement __floatdisf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatdisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// -/* Returns: convert a to a float, rounding toward even.*/ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * di_int is a 64 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// di_int is a 64 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm #include "int_lib.h" -COMPILER_RT_ABI float -__floatdisf(di_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(di_int) * CHAR_BIT; - const di_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = ((du_int)a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((du_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floatdisf(di_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(di_int) * CHAR_BIT; + const di_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = ((du_int)a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((du_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((su_int)s & 0x80000000) | /* sign */ - ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((su_int)s & 0x80000000) | // sign + ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_l2f(di_int a) { - return __floatdisf(a); -} +AEABI_RTABI float __aeabi_l2f(di_int a) { return __floatdisf(a); } #else -AEABI_RTABI float __aeabi_l2f(di_int a) COMPILER_RT_ALIAS(__floatdisf); +COMPILER_RT_ALIAS(__floatdisf, __aeabi_l2f) #endif #endif diff --git a/lib/builtins/floatditf.c b/lib/builtins/floatditf.c index cd51dd8aade4..9b07b65825b8 100644 --- a/lib/builtins/floatditf.c +++ b/lib/builtins/floatditf.c @@ -1,9 +1,8 @@ //===-- lib/floatditf.c - integer -> quad-precision conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,32 +18,32 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatditf(di_int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - du_int aAbs = (du_int)a; - if (a < 0) { - sign = signBit; - aAbs = ~(du_int)a + 1U; - } + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + du_int aAbs = (du_int)a; + if (a < 0) { + sign = signBit; + aAbs = ~(du_int)a + 1U; + } - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clzll(aAbs); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clzll(aAbs); + rep_t result; - // Shift a into the significand field, rounding if it is a right-shift - const int shift = significandBits - exponent; - result = (rep_t)aAbs << shift ^ implicitBit; + // Shift a into the significand field, rounding if it is a right-shift + const int shift = significandBits - exponent; + result = (rep_t)aAbs << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #endif diff --git a/lib/builtins/floatdixf.c b/lib/builtins/floatdixf.c index d39e81d7ca75..ad5deb2d4bf5 100644 --- a/lib/builtins/floatdixf.c +++ b/lib/builtins/floatdixf.c @@ -1,46 +1,41 @@ -/* ===-- floatdixf.c - Implement __floatdixf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatdixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatdixf.c - Implement __floatdixf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatdixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * di_int is a 64 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits di_int is a 64 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI long double -__floatdixf(di_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(di_int) * CHAR_BIT; - const di_int s = a >> (N-1); - a = (a ^ s) - s; - int clz = __builtin_clzll(a); - int e = (N - 1) - clz ; /* exponent */ - long_double_bits fb; - fb.u.high.s.low = ((su_int)s & 0x00008000) | /* sign */ - (e + 16383); /* exponent */ - fb.u.low.all = a << clz; /* mantissa */ - return fb.f; +COMPILER_RT_ABI long double __floatdixf(di_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(di_int) * CHAR_BIT; + const di_int s = a >> (N - 1); + a = (a ^ s) - s; + int clz = __builtin_clzll(a); + int e = (N - 1) - clz; // exponent + long_double_bits fb; + fb.u.high.s.low = ((su_int)s & 0x00008000) | // sign + (e + 16383); // exponent + fb.u.low.all = a << clz; // mantissa + return fb.f; } -#endif /* !_ARCH_PPC */ +#endif // !_ARCH_PPC diff --git a/lib/builtins/floatsidf.c b/lib/builtins/floatsidf.c index fe051123ce7c..2c66167d794d 100644 --- a/lib/builtins/floatsidf.c +++ b/lib/builtins/floatsidf.c @@ -1,9 +1,8 @@ //===-- lib/floatsidf.c - integer -> double-precision conversion --*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,44 +17,41 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatsidf(int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); - - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - if (a < 0) { - sign = signBit; - a = -a; - } - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field and clear the implicit bit. Extra - // cast to unsigned int is necessary to get the correct behavior for - // the input INT_MIN. - const int shift = significandBits - exponent; - result = (rep_t)(unsigned int)a << shift ^ implicitBit; - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); +COMPILER_RT_ABI fp_t __floatsidf(int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + if (a < 0) { + sign = signBit; + a = -a; + } + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field and clear the implicit bit. Extra + // cast to unsigned int is necessary to get the correct behavior for + // the input INT_MIN. + const int shift = significandBits - exponent; + result = (rep_t)(unsigned int)a << shift ^ implicitBit; + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_i2d(int a) { - return __floatsidf(a); -} +AEABI_RTABI fp_t __aeabi_i2d(int a) { return __floatsidf(a); } #else -AEABI_RTABI fp_t __aeabi_i2d(int a) COMPILER_RT_ALIAS(__floatsidf); +COMPILER_RT_ALIAS(__floatsidf, __aeabi_i2d) #endif #endif diff --git a/lib/builtins/floatsisf.c b/lib/builtins/floatsisf.c index bf087ee3c295..fe060407755b 100644 --- a/lib/builtins/floatsisf.c +++ b/lib/builtins/floatsisf.c @@ -1,9 +1,8 @@ //===-- lib/floatsisf.c - integer -> single-precision conversion --*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,50 +17,49 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatsisf(int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); - - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - if (a < 0) { - sign = signBit; - a = -a; - } - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field, rounding if it is a right-shift - if (exponent <= significandBits) { - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; - } else { - const int shift = exponent - significandBits; - result = (rep_t)a >> shift ^ implicitBit; - rep_t round = (rep_t)a << (typeWidth - shift); - if (round > signBit) result++; - if (round == signBit) result += result & 1; - } - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); +COMPILER_RT_ABI fp_t __floatsisf(int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + if (a < 0) { + sign = signBit; + a = -a; + } + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field, rounding if it is a right-shift + if (exponent <= significandBits) { + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; + } else { + const int shift = exponent - significandBits; + result = (rep_t)a >> shift ^ implicitBit; + rep_t round = (rep_t)a << (typeWidth - shift); + if (round > signBit) + result++; + if (round == signBit) + result += result & 1; + } + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_i2f(int a) { - return __floatsisf(a); -} +AEABI_RTABI fp_t __aeabi_i2f(int a) { return __floatsisf(a); } #else -AEABI_RTABI fp_t __aeabi_i2f(int a) COMPILER_RT_ALIAS(__floatsisf); +COMPILER_RT_ALIAS(__floatsisf, __aeabi_i2f) #endif #endif diff --git a/lib/builtins/floatsitf.c b/lib/builtins/floatsitf.c index f0abca363b5e..f56063f368d9 100644 --- a/lib/builtins/floatsitf.c +++ b/lib/builtins/floatsitf.c @@ -1,9 +1,8 @@ //===-- lib/floatsitf.c - integer -> quad-precision conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,32 +18,32 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatsitf(int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) - return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // All other cases begin by extracting the sign and absolute value of a - rep_t sign = 0; - unsigned aAbs = (unsigned)a; - if (a < 0) { - sign = signBit; - aAbs = ~(unsigned)a + 1U; - } + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + unsigned aAbs = (unsigned)a; + if (a < 0) { + sign = signBit; + aAbs = ~(unsigned)a + 1U; + } - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(aAbs); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(aAbs); + rep_t result; - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)aAbs << shift ^ implicitBit; + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)aAbs << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - // Insert the sign bit and return - return fromRep(result | sign); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); } #endif diff --git a/lib/builtins/floattidf.c b/lib/builtins/floattidf.c index 2702a3c8a2db..0a1c04bec82e 100644 --- a/lib/builtins/floattidf.c +++ b/lib/builtins/floattidf.c @@ -1,83 +1,73 @@ -/* ===-- floattidf.c - Implement __floattidf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floattidf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floattidf.c - Implement __floattidf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floattidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a double, rounding toward even.*/ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * ti_int is a 128 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// ti_int is a 128 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm -COMPILER_RT_ABI double -__floattidf(ti_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floattidf(ti_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((su_int)s & 0x80000000) | /* sign */ - ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((su_int)s & 0x80000000) | // sign + ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floattisf.c b/lib/builtins/floattisf.c index f1b585f2c326..a8fcdbe14c07 100644 --- a/lib/builtins/floattisf.c +++ b/lib/builtins/floattisf.c @@ -1,82 +1,71 @@ -/* ===-- floattisf.c - Implement __floattisf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floattisf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floattisf.c - Implement __floattisf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floattisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a float, rounding toward even. */ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * ti_int is a 128 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// ti_int is a 128 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm -COMPILER_RT_ABI float -__floattisf(ti_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floattisf(ti_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((tu_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((su_int)s & 0x80000000) | /* sign */ - ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((su_int)s & 0x80000000) | // sign + ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floattitf.c b/lib/builtins/floattitf.c index 994fded3947b..196cbdae14e0 100644 --- a/lib/builtins/floattitf.c +++ b/lib/builtins/floattitf.c @@ -1,9 +1,8 @@ //===-- lib/floattitf.c - int128 -> quad-precision conversion -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,66 +16,63 @@ #include "fp_lib.h" #include "int_lib.h" -/* Returns: convert a ti_int to a fp_t, rounding toward even. */ +// Returns: convert a ti_int to a fp_t, rounding toward even. -/* Assumption: fp_t is a IEEE 128 bit floating point type - * ti_int is a 128 bit integral type - */ +// Assumption: fp_t is a IEEE 128 bit floating point type +// ti_int is a 128 bit integral type -/* seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | - * mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) -COMPILER_RT_ABI fp_t -__floattitf(ti_int a) { - if (a == 0) - return 0.0; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ - } else { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI fp_t __floattitf(ti_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } - long_double_bits fb; - fb.u.high.all = (s & 0x8000000000000000LL) /* sign */ - | (du_int)(e + 16383) << 48 /* exponent */ - | ((a >> 64) & 0x0000ffffffffffffLL); /* significand */ - fb.u.low.all = (du_int)(a); - return fb.f; + long_double_bits fb; + fb.u.high.all = (s & 0x8000000000000000LL) // sign + | (du_int)(e + 16383) << 48 // exponent + | ((a >> 64) & 0x0000ffffffffffffLL); // significand + fb.u.low.all = (du_int)(a); + return fb.f; } #endif diff --git a/lib/builtins/floattixf.c b/lib/builtins/floattixf.c index 1203b3a96e75..23796f1bb56f 100644 --- a/lib/builtins/floattixf.c +++ b/lib/builtins/floattixf.c @@ -1,84 +1,73 @@ -/* ===-- floattixf.c - Implement __floattixf -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floattixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floattixf.c - Implement __floattixf -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floattixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * ti_int is a 128 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits ti_int is a 128 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI long double -__floattixf(ti_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(ti_int) * CHAR_BIT; - const ti_int s = a >> (N-1); - a = (a ^ s) - s; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = ((tu_int)a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI long double __floattixf(ti_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(ti_int) * CHAR_BIT; + const ti_int s = a >> (N - 1); + a = (a ^ s) - s; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = ((tu_int)a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ - } - long_double_bits fb; - fb.u.high.s.low = ((su_int)s & 0x8000) | /* sign */ - (e + 16383); /* exponent */ - fb.u.low.all = (du_int)a; /* mantissa */ - return fb.f; + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } + long_double_bits fb; + fb.u.high.s.low = ((su_int)s & 0x8000) | // sign + (e + 16383); // exponent + fb.u.low.all = (du_int)a; // mantissa + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatundidf.c b/lib/builtins/floatundidf.c index 8bc2a096324f..e7c6aae5ce38 100644 --- a/lib/builtins/floatundidf.c +++ b/lib/builtins/floatundidf.c @@ -1,114 +1,106 @@ -/* ===-- floatundidf.c - Implement __floatundidf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatundidf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatundidf.c - Implement __floatundidf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatundidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// -/* Returns: convert a to a double, rounding toward even. */ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * du_int is a 64 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// du_int is a 64 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm #include "int_lib.h" #ifndef __SOFT_FP__ -/* Support for systems that have hardware floating-point; we'll set the inexact flag - * as a side-effect of this computation. - */ +// Support for systems that have hardware floating-point; we'll set the inexact +// flag as a side-effect of this computation. -COMPILER_RT_ABI double -__floatundidf(du_int a) -{ - static const double twop52 = 4503599627370496.0; // 0x1.0p52 - static const double twop84 = 19342813113834066795298816.0; // 0x1.0p84 - static const double twop84_plus_twop52 = 19342813118337666422669312.0; // 0x1.00000001p84 +COMPILER_RT_ABI double __floatundidf(du_int a) { + static const double twop52 = 4503599627370496.0; // 0x1.0p52 + static const double twop84 = 19342813113834066795298816.0; // 0x1.0p84 + static const double twop84_plus_twop52 = + 19342813118337666422669312.0; // 0x1.00000001p84 - union { uint64_t x; double d; } high = { .d = twop84 }; - union { uint64_t x; double d; } low = { .d = twop52 }; + union { + uint64_t x; + double d; + } high = {.d = twop84}; + union { + uint64_t x; + double d; + } low = {.d = twop52}; - high.x |= a >> 32; - low.x |= a & UINT64_C(0x00000000ffffffff); + high.x |= a >> 32; + low.x |= a & UINT64_C(0x00000000ffffffff); - const double result = (high.d - twop84_plus_twop52) + low.d; - return result; + const double result = (high.d - twop84_plus_twop52) + low.d; + return result; } #else -/* Support for systems that don't have hardware floating-point; there are no flags to - * set, and we don't want to code-gen to an unknown soft-float implementation. - */ +// Support for systems that don't have hardware floating-point; there are no +// flags to set, and we don't want to code-gen to an unknown soft-float +// implementation. -COMPILER_RT_ABI double -__floatundidf(du_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(du_int) * CHAR_BIT; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((du_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floatundidf(du_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(du_int) * CHAR_BIT; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((du_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } #endif #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI double __aeabi_ul2d(du_int a) { - return __floatundidf(a); -} +AEABI_RTABI double __aeabi_ul2d(du_int a) { return __floatundidf(a); } #else -AEABI_RTABI double __aeabi_ul2d(du_int a) COMPILER_RT_ALIAS(__floatundidf); +COMPILER_RT_ALIAS(__floatundidf, __aeabi_ul2d) #endif #endif diff --git a/lib/builtins/floatundisf.c b/lib/builtins/floatundisf.c index 844786ea7777..87841b761ded 100644 --- a/lib/builtins/floatundisf.c +++ b/lib/builtins/floatundisf.c @@ -1,85 +1,72 @@ -/*===-- floatundisf.c - Implement __floatundisf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatundisf for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- floatundisf.c - Implement __floatundisf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatundisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// -/* Returns: convert a to a float, rounding toward even. */ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * du_int is a 64 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// du_int is a 64 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm #include "int_lib.h" -COMPILER_RT_ABI float -__floatundisf(du_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(du_int) * CHAR_BIT; - int sd = N - __builtin_clzll(a); /* number of significant digits */ - int e = sd - 1; /* 8 exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((du_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floatundisf(du_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(du_int) * CHAR_BIT; + int sd = N - __builtin_clzll(a); // number of significant digits + int e = sd - 1; // 8 exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((du_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_ul2f(du_int a) { - return __floatundisf(a); -} +AEABI_RTABI float __aeabi_ul2f(du_int a) { return __floatundisf(a); } #else -AEABI_RTABI float __aeabi_ul2f(du_int a) COMPILER_RT_ALIAS(__floatundisf); +COMPILER_RT_ALIAS(__floatundisf, __aeabi_ul2f) #endif #endif diff --git a/lib/builtins/floatunditf.c b/lib/builtins/floatunditf.c index 8098e95e82bc..8d310851e179 100644 --- a/lib/builtins/floatunditf.c +++ b/lib/builtins/floatunditf.c @@ -1,9 +1,8 @@ //===-- lib/floatunditf.c - uint -> quad-precision conversion -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,22 +18,23 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatunditf(du_int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clzll(a); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clzll(a); + rep_t result; - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #endif diff --git a/lib/builtins/floatundixf.c b/lib/builtins/floatundixf.c index ca5e06d64dcd..85264adac197 100644 --- a/lib/builtins/floatundixf.c +++ b/lib/builtins/floatundixf.c @@ -1,42 +1,37 @@ -/* ===-- floatundixf.c - Implement __floatundixf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatundixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatundixf.c - Implement __floatundixf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatundixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * du_int is a 64 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits du_int is a 64 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ -COMPILER_RT_ABI long double -__floatundixf(du_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(du_int) * CHAR_BIT; - int clz = __builtin_clzll(a); - int e = (N - 1) - clz ; /* exponent */ - long_double_bits fb; - fb.u.high.s.low = (e + 16383); /* exponent */ - fb.u.low.all = a << clz; /* mantissa */ - return fb.f; +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm +COMPILER_RT_ABI long double __floatundixf(du_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(du_int) * CHAR_BIT; + int clz = __builtin_clzll(a); + int e = (N - 1) - clz; // exponent + long_double_bits fb; + fb.u.high.s.low = (e + 16383); // exponent + fb.u.low.all = a << clz; // mantissa + return fb.f; } -#endif /* _ARCH_PPC */ +#endif // _ARCH_PPC diff --git a/lib/builtins/floatunsidf.c b/lib/builtins/floatunsidf.c index 75cf6b9177df..2c01c3041434 100644 --- a/lib/builtins/floatunsidf.c +++ b/lib/builtins/floatunsidf.c @@ -1,9 +1,8 @@ //===-- lib/floatunsidf.c - uint -> double-precision conversion ---*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,33 +17,31 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatunsidf(unsigned int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); +COMPILER_RT_ABI fp_t __floatunsidf(unsigned int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) { - return __floatunsidf(a); -} +AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) { return __floatunsidf(a); } #else -AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) COMPILER_RT_ALIAS(__floatunsidf); +COMPILER_RT_ALIAS(__floatunsidf, __aeabi_ui2d) #endif #endif diff --git a/lib/builtins/floatunsisf.c b/lib/builtins/floatunsisf.c index 29525ccedbbe..33a1b5ae2a63 100644 --- a/lib/builtins/floatunsisf.c +++ b/lib/builtins/floatunsisf.c @@ -1,9 +1,8 @@ //===-- lib/floatunsisf.c - uint -> single-precision conversion ---*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,41 +17,41 @@ #include "int_lib.h" -COMPILER_RT_ABI fp_t -__floatunsisf(unsigned int a) { - - const int aWidth = sizeof a * CHAR_BIT; - - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); - - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; - - // Shift a into the significand field, rounding if it is a right-shift - if (exponent <= significandBits) { - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; - } else { - const int shift = exponent - significandBits; - result = (rep_t)a >> shift ^ implicitBit; - rep_t round = (rep_t)a << (typeWidth - shift); - if (round > signBit) result++; - if (round == signBit) result += result & 1; - } - - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); +COMPILER_RT_ABI fp_t __floatunsisf(unsigned int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; + + // Shift a into the significand field, rounding if it is a right-shift + if (exponent <= significandBits) { + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; + } else { + const int shift = exponent - significandBits; + result = (rep_t)a >> shift ^ implicitBit; + rep_t round = (rep_t)a << (typeWidth - shift); + if (round > signBit) + result++; + if (round == signBit) + result += result & 1; + } + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) { - return __floatunsisf(a); -} +AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) { return __floatunsisf(a); } #else -AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) COMPILER_RT_ALIAS(__floatunsisf); +COMPILER_RT_ALIAS(__floatunsisf, __aeabi_ui2f) #endif #endif diff --git a/lib/builtins/floatunsitf.c b/lib/builtins/floatunsitf.c index 1cd1842e709a..a4bf0f65fe1c 100644 --- a/lib/builtins/floatunsitf.c +++ b/lib/builtins/floatunsitf.c @@ -1,9 +1,8 @@ //===-- lib/floatunsitf.c - uint -> quad-precision conversion -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,22 +18,23 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) COMPILER_RT_ABI fp_t __floatunsitf(unsigned int a) { - const int aWidth = sizeof a * CHAR_BIT; + const int aWidth = sizeof a * CHAR_BIT; - // Handle zero as a special case to protect clz - if (a == 0) return fromRep(0); + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); - // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); - rep_t result; + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clz(a); + rep_t result; - // Shift a into the significand field and clear the implicit bit. - const int shift = significandBits - exponent; - result = (rep_t)a << shift ^ implicitBit; + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; - // Insert the exponent - result += (rep_t)(exponent + exponentBias) << significandBits; - return fromRep(result); + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); } #endif diff --git a/lib/builtins/floatuntidf.c b/lib/builtins/floatuntidf.c index 960265d80772..e69e65c1ace4 100644 --- a/lib/builtins/floatuntidf.c +++ b/lib/builtins/floatuntidf.c @@ -1,80 +1,70 @@ -/* ===-- floatuntidf.c - Implement __floatuntidf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatuntidf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatuntidf.c - Implement __floatuntidf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatuntidf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a double, rounding toward even. */ +// Returns: convert a to a double, rounding toward even. -/* Assumption: double is a IEEE 64 bit floating point type - * tu_int is a 128 bit integral type - */ +// Assumption: double is a IEEE 64 bit floating point type +// tu_int is a 128 bit integral type -/* seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm -COMPILER_RT_ABI double -__floatuntidf(tu_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > DBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit DBL_MANT_DIG-1 bits to the right of 1 - * Q = bit DBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case DBL_MANT_DIG + 1: - a <<= 1; - break; - case DBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (DBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << DBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to DBL_MANT_DIG bits */ +COMPILER_RT_ABI double __floatuntidf(tu_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > DBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit DBL_MANT_DIG-1 bits to the right of 1 + // Q = bit DBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case DBL_MANT_DIG + 1: + a <<= 1; + break; + case DBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (DBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << DBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (DBL_MANT_DIG - sd); - /* a is now rounded to DBL_MANT_DIG bits */ - } - double_bits fb; - fb.u.s.high = ((e + 1023) << 20) | /* exponent */ - ((su_int)(a >> 32) & 0x000FFFFF); /* mantissa-high */ - fb.u.s.low = (su_int)a; /* mantissa-low */ - return fb.f; + // a is now rounded to DBL_MANT_DIG bits + } else { + a <<= (DBL_MANT_DIG - sd); + // a is now rounded to DBL_MANT_DIG bits + } + double_bits fb; + fb.u.s.high = ((e + 1023) << 20) | // exponent + ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high + fb.u.s.low = (su_int)a; // mantissa-low + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatuntisf.c b/lib/builtins/floatuntisf.c index c0dd0275ddba..9dec0ab5c58f 100644 --- a/lib/builtins/floatuntisf.c +++ b/lib/builtins/floatuntisf.c @@ -1,79 +1,68 @@ -/* ===-- floatuntisf.c - Implement __floatuntisf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatuntisf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatuntisf.c - Implement __floatuntisf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatuntisf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a float, rounding toward even. */ +// Returns: convert a to a float, rounding toward even. -/* Assumption: float is a IEEE 32 bit floating point type - * tu_int is a 128 bit integral type - */ +// Assumption: float is a IEEE 32 bit floating point type +// tu_int is a 128 bit integral type -/* seee eeee emmm mmmm mmmm mmmm mmmm mmmm */ +// seee eeee emmm mmmm mmmm mmmm mmmm mmmm -COMPILER_RT_ABI float -__floatuntisf(tu_int a) -{ - if (a == 0) - return 0.0F; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > FLT_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit FLT_MANT_DIG-1 bits to the right of 1 - * Q = bit FLT_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case FLT_MANT_DIG + 1: - a <<= 1; - break; - case FLT_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (FLT_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << FLT_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to FLT_MANT_DIG bits */ +COMPILER_RT_ABI float __floatuntisf(tu_int a) { + if (a == 0) + return 0.0F; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > FLT_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit FLT_MANT_DIG-1 bits to the right of 1 + // Q = bit FLT_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case FLT_MANT_DIG + 1: + a <<= 1; + break; + case FLT_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (FLT_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits + if (a & ((tu_int)1 << FLT_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (FLT_MANT_DIG - sd); - /* a is now rounded to FLT_MANT_DIG bits */ - } - float_bits fb; - fb.u = ((e + 127) << 23) | /* exponent */ - ((su_int)a & 0x007FFFFF); /* mantissa */ - return fb.f; + // a is now rounded to FLT_MANT_DIG bits + } else { + a <<= (FLT_MANT_DIG - sd); + // a is now rounded to FLT_MANT_DIG bits + } + float_bits fb; + fb.u = ((e + 127) << 23) | // exponent + ((su_int)a & 0x007FFFFF); // mantissa + return fb.f; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/floatuntitf.c b/lib/builtins/floatuntitf.c index e2518c93f234..d308d3118d03 100644 --- a/lib/builtins/floatuntitf.c +++ b/lib/builtins/floatuntitf.c @@ -1,9 +1,8 @@ //===-- lib/floatuntitf.c - uint128 -> quad-precision conversion --*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,63 +16,60 @@ #include "fp_lib.h" #include "int_lib.h" -/* Returns: convert a tu_int to a fp_t, rounding toward even. */ +// Returns: convert a tu_int to a fp_t, rounding toward even. -/* Assumption: fp_t is a IEEE 128 bit floating point type - * tu_int is a 128 bit integral type - */ +// Assumption: fp_t is a IEEE 128 bit floating point type +// tu_int is a 128 bit integral type -/* seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | - * mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// seee eeee eeee eeee mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm +// mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) -COMPILER_RT_ABI fp_t -__floatuntitf(tu_int a) { - if (a == 0) - return 0.0; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ - } else { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI fp_t __floatuntitf(tu_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } - long_double_bits fb; - fb.u.high.all = (du_int)(e + 16383) << 48 /* exponent */ - | ((a >> 64) & 0x0000ffffffffffffLL); /* significand */ - fb.u.low.all = (du_int)(a); - return fb.f; + long_double_bits fb; + fb.u.high.all = (du_int)(e + 16383) << 48 // exponent + | ((a >> 64) & 0x0000ffffffffffffLL); // significand + fb.u.low.all = (du_int)(a); + return fb.f; } #endif diff --git a/lib/builtins/floatuntixf.c b/lib/builtins/floatuntixf.c index ea81cb1bcdae..efd8a27a0875 100644 --- a/lib/builtins/floatuntixf.c +++ b/lib/builtins/floatuntixf.c @@ -1,81 +1,70 @@ -/* ===-- floatuntixf.c - Implement __floatuntixf ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __floatuntixf for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- floatuntixf.c - Implement __floatuntixf ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __floatuntixf for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: convert a to a long double, rounding toward even. */ +// Returns: convert a to a long double, rounding toward even. -/* Assumption: long double is a IEEE 80 bit floating point type padded to 128 bits - * tu_int is a 128 bit integral type - */ +// Assumption: long double is a IEEE 80 bit floating point type padded to 128 +// bits tu_int is a 128 bit integral type -/* gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee eeee | - * 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm - */ +// gggg gggg gggg gggg gggg gggg gggg gggg | gggg gggg gggg gggg seee eeee eeee +// eeee | 1mmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm +// mmmm mmmm mmmm -COMPILER_RT_ABI long double -__floatuntixf(tu_int a) -{ - if (a == 0) - return 0.0; - const unsigned N = sizeof(tu_int) * CHAR_BIT; - int sd = N - __clzti2(a); /* number of significant digits */ - int e = sd - 1; /* exponent */ - if (sd > LDBL_MANT_DIG) - { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit LDBL_MANT_DIG-1 bits to the right of 1 - * Q = bit LDBL_MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - switch (sd) - { - case LDBL_MANT_DIG + 1: - a <<= 1; - break; - case LDBL_MANT_DIG + 2: - break; - default: - a = (a >> (sd - (LDBL_MANT_DIG+2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG+2) - sd))) != 0); - }; - /* finish: */ - a |= (a & 4) != 0; /* Or P into R */ - ++a; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - /* a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits */ - if (a & ((tu_int)1 << LDBL_MANT_DIG)) - { - a >>= 1; - ++e; - } - /* a is now rounded to LDBL_MANT_DIG bits */ +COMPILER_RT_ABI long double __floatuntixf(tu_int a) { + if (a == 0) + return 0.0; + const unsigned N = sizeof(tu_int) * CHAR_BIT; + int sd = N - __clzti2(a); // number of significant digits + int e = sd - 1; // exponent + if (sd > LDBL_MANT_DIG) { + // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx + // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR + // 12345678901234567890123456 + // 1 = msb 1 bit + // P = bit LDBL_MANT_DIG-1 bits to the right of 1 + // Q = bit LDBL_MANT_DIG bits to the right of 1 + // R = "or" of all bits to the right of Q + switch (sd) { + case LDBL_MANT_DIG + 1: + a <<= 1; + break; + case LDBL_MANT_DIG + 2: + break; + default: + a = (a >> (sd - (LDBL_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + }; + // finish: + a |= (a & 4) != 0; // Or P into R + ++a; // round - this step may add a significant bit + a >>= 2; // dump Q and R + // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits + if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + a >>= 1; + ++e; } - else - { - a <<= (LDBL_MANT_DIG - sd); - /* a is now rounded to LDBL_MANT_DIG bits */ - } - long_double_bits fb; - fb.u.high.s.low = (e + 16383); /* exponent */ - fb.u.low.all = (du_int)a; /* mantissa */ - return fb.f; + // a is now rounded to LDBL_MANT_DIG bits + } else { + a <<= (LDBL_MANT_DIG - sd); + // a is now rounded to LDBL_MANT_DIG bits + } + long_double_bits fb; + fb.u.high.s.low = (e + 16383); // exponent + fb.u.low.all = (du_int)a; // mantissa + return fb.f; } #endif diff --git a/lib/builtins/fp_add_impl.inc b/lib/builtins/fp_add_impl.inc index b47be1b648e6..da8639341703 100644 --- a/lib/builtins/fp_add_impl.inc +++ b/lib/builtins/fp_add_impl.inc @@ -1,9 +1,8 @@ //===----- lib/fp_add_impl.inc - floaing point addition -----------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,130 +14,144 @@ #include "fp_lib.h" static __inline fp_t __addXf3__(fp_t a, fp_t b) { - rep_t aRep = toRep(a); - rep_t bRep = toRep(b); - const rep_t aAbs = aRep & absMask; - const rep_t bAbs = bRep & absMask; - - // Detect if a or b is zero, infinity, or NaN. - if (aAbs - REP_C(1) >= infRep - REP_C(1) || - bAbs - REP_C(1) >= infRep - REP_C(1)) { - // NaN + anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything + NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // +/-infinity + -/+infinity = qNaN - if ((toRep(a) ^ toRep(b)) == signBit) return fromRep(qnanRep); - // +/-infinity + anything remaining = +/- infinity - else return a; - } - - // anything remaining + +/-infinity = +/-infinity - if (bAbs == infRep) return b; - - // zero + anything = anything - if (!aAbs) { - // but we need to get the sign right for zero + zero - if (!bAbs) return fromRep(toRep(a) & toRep(b)); - else return b; - } - - // anything + zero = anything - if (!bAbs) return a; + rep_t aRep = toRep(a); + rep_t bRep = toRep(b); + const rep_t aAbs = aRep & absMask; + const rep_t bAbs = bRep & absMask; + + // Detect if a or b is zero, infinity, or NaN. + if (aAbs - REP_C(1) >= infRep - REP_C(1) || + bAbs - REP_C(1) >= infRep - REP_C(1)) { + // NaN + anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything + NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // +/-infinity + -/+infinity = qNaN + if ((toRep(a) ^ toRep(b)) == signBit) + return fromRep(qnanRep); + // +/-infinity + anything remaining = +/- infinity + else + return a; } - // Swap a and b if necessary so that a has the larger absolute value. - if (bAbs > aAbs) { - const rep_t temp = aRep; - aRep = bRep; - bRep = temp; + // anything remaining + +/-infinity = +/-infinity + if (bAbs == infRep) + return b; + + // zero + anything = anything + if (!aAbs) { + // We need to get the sign right for zero + zero. + if (!bAbs) + return fromRep(toRep(a) & toRep(b)); + else + return b; } - // Extract the exponent and significand from the (possibly swapped) a and b. - int aExponent = aRep >> significandBits & maxExponent; - int bExponent = bRep >> significandBits & maxExponent; - rep_t aSignificand = aRep & significandMask; - rep_t bSignificand = bRep & significandMask; - - // Normalize any denormals, and adjust the exponent accordingly. - if (aExponent == 0) aExponent = normalize(&aSignificand); - if (bExponent == 0) bExponent = normalize(&bSignificand); - - // The sign of the result is the sign of the larger operand, a. If they - // have opposite signs, we are performing a subtraction; otherwise addition. - const rep_t resultSign = aRep & signBit; - const bool subtraction = (aRep ^ bRep) & signBit; - - // Shift the significands to give us round, guard and sticky, and or in the - // implicit significand bit. (If we fell through from the denormal path it - // was already set by normalize( ), but setting it twice won't hurt - // anything.) - aSignificand = (aSignificand | implicitBit) << 3; - bSignificand = (bSignificand | implicitBit) << 3; - - // Shift the significand of b by the difference in exponents, with a sticky - // bottom bit to get rounding correct. - const unsigned int align = aExponent - bExponent; - if (align) { - if (align < typeWidth) { - const bool sticky = bSignificand << (typeWidth - align); - bSignificand = bSignificand >> align | sticky; - } else { - bSignificand = 1; // sticky; b is known to be non-zero. - } - } - if (subtraction) { - aSignificand -= bSignificand; - // If a == -b, return +zero. - if (aSignificand == 0) return fromRep(0); - - // If partial cancellation occured, we need to left-shift the result - // and adjust the exponent: - if (aSignificand < implicitBit << 3) { - const int shift = rep_clz(aSignificand) - rep_clz(implicitBit << 3); - aSignificand <<= shift; - aExponent -= shift; - } + // anything + zero = anything + if (!bAbs) + return a; + } + + // Swap a and b if necessary so that a has the larger absolute value. + if (bAbs > aAbs) { + const rep_t temp = aRep; + aRep = bRep; + bRep = temp; + } + + // Extract the exponent and significand from the (possibly swapped) a and b. + int aExponent = aRep >> significandBits & maxExponent; + int bExponent = bRep >> significandBits & maxExponent; + rep_t aSignificand = aRep & significandMask; + rep_t bSignificand = bRep & significandMask; + + // Normalize any denormals, and adjust the exponent accordingly. + if (aExponent == 0) + aExponent = normalize(&aSignificand); + if (bExponent == 0) + bExponent = normalize(&bSignificand); + + // The sign of the result is the sign of the larger operand, a. If they + // have opposite signs, we are performing a subtraction. Otherwise, we + // perform addition. + const rep_t resultSign = aRep & signBit; + const bool subtraction = (aRep ^ bRep) & signBit; + + // Shift the significands to give us round, guard and sticky, and set the + // implicit significand bit. If we fell through from the denormal path it + // was already set by normalize( ), but setting it twice won't hurt + // anything. + aSignificand = (aSignificand | implicitBit) << 3; + bSignificand = (bSignificand | implicitBit) << 3; + + // Shift the significand of b by the difference in exponents, with a sticky + // bottom bit to get rounding correct. + const unsigned int align = aExponent - bExponent; + if (align) { + if (align < typeWidth) { + const bool sticky = bSignificand << (typeWidth - align); + bSignificand = bSignificand >> align | sticky; + } else { + bSignificand = 1; // Set the sticky bit. b is known to be non-zero. } - else /* addition */ { - aSignificand += bSignificand; - - // If the addition carried up, we need to right-shift the result and - // adjust the exponent: - if (aSignificand & implicitBit << 4) { - const bool sticky = aSignificand & 1; - aSignificand = aSignificand >> 1 | sticky; - aExponent += 1; - } + } + if (subtraction) { + aSignificand -= bSignificand; + // If a == -b, return +zero. + if (aSignificand == 0) + return fromRep(0); + + // If partial cancellation occured, we need to left-shift the result + // and adjust the exponent. + if (aSignificand < implicitBit << 3) { + const int shift = rep_clz(aSignificand) - rep_clz(implicitBit << 3); + aSignificand <<= shift; + aExponent -= shift; } - - // If we have overflowed the type, return +/- infinity: - if (aExponent >= maxExponent) return fromRep(infRep | resultSign); - - if (aExponent <= 0) { - // Result is denormal before rounding; the exponent is zero and we - // need to shift the significand. - const int shift = 1 - aExponent; - const bool sticky = aSignificand << (typeWidth - shift); - aSignificand = aSignificand >> shift | sticky; - aExponent = 0; + } else /* addition */ { + aSignificand += bSignificand; + + // If the addition carried up, we need to right-shift the result and + // adjust the exponent. + if (aSignificand & implicitBit << 4) { + const bool sticky = aSignificand & 1; + aSignificand = aSignificand >> 1 | sticky; + aExponent += 1; } - - // Low three bits are round, guard, and sticky. - const int roundGuardSticky = aSignificand & 0x7; - - // Shift the significand into place, and mask off the implicit bit. - rep_t result = aSignificand >> 3 & significandMask; - - // Insert the exponent and sign. - result |= (rep_t)aExponent << significandBits; - result |= resultSign; - - // Final rounding. The result may overflow to infinity, but that is the - // correct result in that case. - if (roundGuardSticky > 0x4) result++; - if (roundGuardSticky == 0x4) result += result & 1; - return fromRep(result); + } + + // If we have overflowed the type, return +/- infinity. + if (aExponent >= maxExponent) + return fromRep(infRep | resultSign); + + if (aExponent <= 0) { + // The result is denormal before rounding. The exponent is zero and we + // need to shift the significand. + const int shift = 1 - aExponent; + const bool sticky = aSignificand << (typeWidth - shift); + aSignificand = aSignificand >> shift | sticky; + aExponent = 0; + } + + // Low three bits are round, guard, and sticky. + const int roundGuardSticky = aSignificand & 0x7; + + // Shift the significand into place, and mask off the implicit bit. + rep_t result = aSignificand >> 3 & significandMask; + + // Insert the exponent and sign. + result |= (rep_t)aExponent << significandBits; + result |= resultSign; + + // Perform the final rounding. The result may overflow to infinity, but + // that is the correct result in that case. + if (roundGuardSticky > 0x4) + result++; + if (roundGuardSticky == 0x4) + result += result & 1; + return fromRep(result); } diff --git a/lib/builtins/fp_extend.h b/lib/builtins/fp_extend.h index 6d95a0680709..d2083c426722 100644 --- a/lib/builtins/fp_extend.h +++ b/lib/builtins/fp_extend.h @@ -1,9 +1,9 @@ -//===-lib/fp_extend.h - low precision -> high precision conversion -*- C -*-===// +//===-lib/fp_extend.h - low precision -> high precision conversion -*- C +//-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,12 +30,12 @@ typedef uint64_t src_rep_t; static const int srcSigBits = 52; static __inline int src_rep_t_clz(src_rep_t a) { #if defined __LP64__ - return __builtin_clzl(a); + return __builtin_clzl(a); #else - if (a & REP_C(0xffffffff00000000)) - return __builtin_clz(a >> 32); - else - return 32 + __builtin_clz(a & REP_C(0xffffffff)); + if (a & REP_C(0xffffffff00000000)) + return __builtin_clz(a >> 32); + else + return 32 + __builtin_clz(a & REP_C(0xffffffff)); #endif } @@ -48,7 +48,7 @@ static const int srcSigBits = 10; #else #error Source should be half, single, or double precision! -#endif //end source precision +#endif // end source precision #if defined DST_SINGLE typedef float dst_t; @@ -70,20 +70,26 @@ static const int dstSigBits = 112; #else #error Destination should be single, double, or quad precision! -#endif //end destination precision +#endif // end destination precision // End of specialization parameters. Two helper routines for conversion to and // from the representation of floating-point data as integer values follow. static __inline src_rep_t srcToRep(src_t x) { - const union { src_t f; src_rep_t i; } rep = {.f = x}; - return rep.i; + const union { + src_t f; + src_rep_t i; + } rep = {.f = x}; + return rep.i; } static __inline dst_t dstFromRep(dst_rep_t x) { - const union { dst_t f; dst_rep_t i; } rep = {.i = x}; - return rep.f; + const union { + dst_t f; + dst_rep_t i; + } rep = {.i = x}; + return rep.f; } // End helper routines. Conversion implementation follows. -#endif //FP_EXTEND_HEADER +#endif // FP_EXTEND_HEADER diff --git a/lib/builtins/fp_extend_impl.inc b/lib/builtins/fp_extend_impl.inc index b785cc7687ad..d1c9c02a00c5 100644 --- a/lib/builtins/fp_extend_impl.inc +++ b/lib/builtins/fp_extend_impl.inc @@ -1,9 +1,8 @@ //=-lib/fp_extend_impl.inc - low precision -> high precision conversion -*-- -// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,81 +27,81 @@ // // Finally, the following assumptions are made: // -// 1. floating-point types and integer types have the same endianness on the -// target platform +// 1. Floating-point types and integer types have the same endianness on the +// target platform. // -// 2. quiet NaNs, if supported, are indicated by the leading bit of the -// significand field being set +// 2. Quiet NaNs, if supported, are indicated by the leading bit of the +// significand field being set. // //===----------------------------------------------------------------------===// #include "fp_extend.h" static __inline dst_t __extendXfYf2__(src_t a) { - // Various constants whose values follow from the type parameters. - // Any reasonable optimizer will fold and propagate all of these. - const int srcBits = sizeof(src_t)*CHAR_BIT; - const int srcExpBits = srcBits - srcSigBits - 1; - const int srcInfExp = (1 << srcExpBits) - 1; - const int srcExpBias = srcInfExp >> 1; + // Various constants whose values follow from the type parameters. + // Any reasonable optimizer will fold and propagate all of these. + const int srcBits = sizeof(src_t) * CHAR_BIT; + const int srcExpBits = srcBits - srcSigBits - 1; + const int srcInfExp = (1 << srcExpBits) - 1; + const int srcExpBias = srcInfExp >> 1; - const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; - const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; - const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); - const src_rep_t srcAbsMask = srcSignMask - 1; - const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); - const src_rep_t srcNaNCode = srcQNaN - 1; + const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; + const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; + const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); + const src_rep_t srcAbsMask = srcSignMask - 1; + const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); + const src_rep_t srcNaNCode = srcQNaN - 1; - const int dstBits = sizeof(dst_t)*CHAR_BIT; - const int dstExpBits = dstBits - dstSigBits - 1; - const int dstInfExp = (1 << dstExpBits) - 1; - const int dstExpBias = dstInfExp >> 1; + const int dstBits = sizeof(dst_t) * CHAR_BIT; + const int dstExpBits = dstBits - dstSigBits - 1; + const int dstInfExp = (1 << dstExpBits) - 1; + const int dstExpBias = dstInfExp >> 1; - const dst_rep_t dstMinNormal = DST_REP_C(1) << dstSigBits; + const dst_rep_t dstMinNormal = DST_REP_C(1) << dstSigBits; - // Break a into a sign and representation of the absolute value - const src_rep_t aRep = srcToRep(a); - const src_rep_t aAbs = aRep & srcAbsMask; - const src_rep_t sign = aRep & srcSignMask; - dst_rep_t absResult; + // Break a into a sign and representation of the absolute value. + const src_rep_t aRep = srcToRep(a); + const src_rep_t aAbs = aRep & srcAbsMask; + const src_rep_t sign = aRep & srcSignMask; + dst_rep_t absResult; - // If sizeof(src_rep_t) < sizeof(int), the subtraction result is promoted - // to (signed) int. To avoid that, explicitly cast to src_rep_t. - if ((src_rep_t)(aAbs - srcMinNormal) < srcInfinity - srcMinNormal) { - // a is a normal number. - // Extend to the destination type by shifting the significand and - // exponent into the proper position and rebiasing the exponent. - absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits); - absResult += (dst_rep_t)(dstExpBias - srcExpBias) << dstSigBits; - } + // If sizeof(src_rep_t) < sizeof(int), the subtraction result is promoted + // to (signed) int. To avoid that, explicitly cast to src_rep_t. + if ((src_rep_t)(aAbs - srcMinNormal) < srcInfinity - srcMinNormal) { + // a is a normal number. + // Extend to the destination type by shifting the significand and + // exponent into the proper position and rebiasing the exponent. + absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits); + absResult += (dst_rep_t)(dstExpBias - srcExpBias) << dstSigBits; + } - else if (aAbs >= srcInfinity) { - // a is NaN or infinity. - // Conjure the result by beginning with infinity, then setting the qNaN - // bit (if needed) and right-aligning the rest of the trailing NaN - // payload field. - absResult = (dst_rep_t)dstInfExp << dstSigBits; - absResult |= (dst_rep_t)(aAbs & srcQNaN) << (dstSigBits - srcSigBits); - absResult |= (dst_rep_t)(aAbs & srcNaNCode) << (dstSigBits - srcSigBits); - } + else if (aAbs >= srcInfinity) { + // a is NaN or infinity. + // Conjure the result by beginning with infinity, then setting the qNaN + // bit (if needed) and right-aligning the rest of the trailing NaN + // payload field. + absResult = (dst_rep_t)dstInfExp << dstSigBits; + absResult |= (dst_rep_t)(aAbs & srcQNaN) << (dstSigBits - srcSigBits); + absResult |= (dst_rep_t)(aAbs & srcNaNCode) << (dstSigBits - srcSigBits); + } - else if (aAbs) { - // a is denormal. - // renormalize the significand and clear the leading bit, then insert - // the correct adjusted exponent in the destination type. - const int scale = src_rep_t_clz(aAbs) - src_rep_t_clz(srcMinNormal); - absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits + scale); - absResult ^= dstMinNormal; - const int resultExponent = dstExpBias - srcExpBias - scale + 1; - absResult |= (dst_rep_t)resultExponent << dstSigBits; - } + else if (aAbs) { + // a is denormal. + // renormalize the significand and clear the leading bit, then insert + // the correct adjusted exponent in the destination type. + const int scale = src_rep_t_clz(aAbs) - src_rep_t_clz(srcMinNormal); + absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits + scale); + absResult ^= dstMinNormal; + const int resultExponent = dstExpBias - srcExpBias - scale + 1; + absResult |= (dst_rep_t)resultExponent << dstSigBits; + } - else { - // a is zero. - absResult = 0; - } + else { + // a is zero. + absResult = 0; + } - // Apply the signbit to (dst_t)abs(a). - const dst_rep_t result = absResult | (dst_rep_t)sign << (dstBits - srcBits); - return dstFromRep(result); + // Apply the signbit to the absolute value. + const dst_rep_t result = absResult | (dst_rep_t)sign << (dstBits - srcBits); + return dstFromRep(result); } diff --git a/lib/builtins/fp_fixint_impl.inc b/lib/builtins/fp_fixint_impl.inc index da70d4d39301..2196d712f05f 100644 --- a/lib/builtins/fp_fixint_impl.inc +++ b/lib/builtins/fp_fixint_impl.inc @@ -1,9 +1,8 @@ //===-- lib/fixdfsi.c - Double-precision -> integer conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,27 +14,27 @@ #include "fp_lib.h" static __inline fixint_t __fixint(fp_t a) { - const fixint_t fixint_max = (fixint_t)((~(fixuint_t)0) / 2); - const fixint_t fixint_min = -fixint_max - 1; - // Break a into sign, exponent, significand - const rep_t aRep = toRep(a); - const rep_t aAbs = aRep & absMask; - const fixint_t sign = aRep & signBit ? -1 : 1; - const int exponent = (aAbs >> significandBits) - exponentBias; - const rep_t significand = (aAbs & significandMask) | implicitBit; + const fixint_t fixint_max = (fixint_t)((~(fixuint_t)0) / 2); + const fixint_t fixint_min = -fixint_max - 1; + // Break a into sign, exponent, significand parts. + const rep_t aRep = toRep(a); + const rep_t aAbs = aRep & absMask; + const fixint_t sign = aRep & signBit ? -1 : 1; + const int exponent = (aAbs >> significandBits) - exponentBias; + const rep_t significand = (aAbs & significandMask) | implicitBit; - // If exponent is negative, the result is zero. - if (exponent < 0) - return 0; + // If exponent is negative, the result is zero. + if (exponent < 0) + return 0; - // If the value is too large for the integer type, saturate. - if ((unsigned)exponent >= sizeof(fixint_t) * CHAR_BIT) - return sign == 1 ? fixint_max : fixint_min; + // If the value is too large for the integer type, saturate. + if ((unsigned)exponent >= sizeof(fixint_t) * CHAR_BIT) + return sign == 1 ? fixint_max : fixint_min; - // If 0 <= exponent < significandBits, right shift to get the result. - // Otherwise, shift left. - if (exponent < significandBits) - return sign * (significand >> (significandBits - exponent)); - else - return sign * ((fixint_t)significand << (exponent - significandBits)); + // If 0 <= exponent < significandBits, right shift to get the result. + // Otherwise, shift left. + if (exponent < significandBits) + return sign * (significand >> (significandBits - exponent)); + else + return sign * ((fixint_t)significand << (exponent - significandBits)); } diff --git a/lib/builtins/fp_fixuint_impl.inc b/lib/builtins/fp_fixuint_impl.inc index d68ccf27a79c..cb2bf54ffaf5 100644 --- a/lib/builtins/fp_fixuint_impl.inc +++ b/lib/builtins/fp_fixuint_impl.inc @@ -1,9 +1,8 @@ //===-- lib/fixdfsi.c - Double-precision -> integer conversion ----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,25 +14,25 @@ #include "fp_lib.h" static __inline fixuint_t __fixuint(fp_t a) { - // Break a into sign, exponent, significand - const rep_t aRep = toRep(a); - const rep_t aAbs = aRep & absMask; - const int sign = aRep & signBit ? -1 : 1; - const int exponent = (aAbs >> significandBits) - exponentBias; - const rep_t significand = (aAbs & significandMask) | implicitBit; + // Break a into sign, exponent, significand parts. + const rep_t aRep = toRep(a); + const rep_t aAbs = aRep & absMask; + const int sign = aRep & signBit ? -1 : 1; + const int exponent = (aAbs >> significandBits) - exponentBias; + const rep_t significand = (aAbs & significandMask) | implicitBit; - // If either the value or the exponent is negative, the result is zero. - if (sign == -1 || exponent < 0) - return 0; + // If either the value or the exponent is negative, the result is zero. + if (sign == -1 || exponent < 0) + return 0; - // If the value is too large for the integer type, saturate. - if ((unsigned)exponent >= sizeof(fixuint_t) * CHAR_BIT) - return ~(fixuint_t)0; + // If the value is too large for the integer type, saturate. + if ((unsigned)exponent >= sizeof(fixuint_t) * CHAR_BIT) + return ~(fixuint_t)0; - // If 0 <= exponent < significandBits, right shift to get the result. - // Otherwise, shift left. - if (exponent < significandBits) - return significand >> (significandBits - exponent); - else - return (fixuint_t)significand << (exponent - significandBits); + // If 0 <= exponent < significandBits, right shift to get the result. + // Otherwise, shift left. + if (exponent < significandBits) + return significand >> (significandBits - exponent); + else + return (fixuint_t)significand << (exponent - significandBits); } diff --git a/lib/builtins/fp_lib.h b/lib/builtins/fp_lib.h index a0e19ab6a8f6..d1a988ea4713 100644 --- a/lib/builtins/fp_lib.h +++ b/lib/builtins/fp_lib.h @@ -1,9 +1,8 @@ //===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,22 +20,22 @@ #ifndef FP_LIB_HEADER #define FP_LIB_HEADER -#include <stdint.h> -#include <stdbool.h> -#include <limits.h> #include "int_lib.h" #include "int_math.h" +#include <limits.h> +#include <stdbool.h> +#include <stdint.h> // x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in // 32-bit mode. #if defined(__FreeBSD__) && defined(__i386__) -# include <sys/param.h> -# if __FreeBSD_version < 903000 // v9.3 -# define uint64_t unsigned long long -# define int64_t long long -# undef UINT64_C -# define UINT64_C(c) (c ## ULL) -# endif +#include <sys/param.h> +#if __FreeBSD_version < 903000 // v9.3 +#define uint64_t unsigned long long +#define int64_t long long +#undef UINT64_C +#define UINT64_C(c) (c##ULL) +#endif #endif #if defined SINGLE_PRECISION @@ -47,15 +46,13 @@ typedef float fp_t; #define REP_C UINT32_C #define significandBits 23 -static __inline int rep_clz(rep_t a) { - return __builtin_clz(a); -} +static __inline int rep_clz(rep_t a) { return __builtin_clz(a); } // 32x32 --> 64 bit multiply static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - const uint64_t product = (uint64_t)a*b; - *hi = product >> 32; - *lo = product; + const uint64_t product = (uint64_t)a * b; + *hi = product >> 32; + *lo = product; } COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b); @@ -69,12 +66,12 @@ typedef double fp_t; static __inline int rep_clz(rep_t a) { #if defined __LP64__ - return __builtin_clzl(a); + return __builtin_clzl(a); #else - if (a & REP_C(0xffffffff00000000)) - return __builtin_clz(a >> 32); - else - return 32 + __builtin_clz(a & REP_C(0xffffffff)); + if (a & REP_C(0xffffffff00000000)) + return __builtin_clz(a >> 32); + else + return 32 + __builtin_clz(a & REP_C(0xffffffff)); #endif } @@ -85,17 +82,17 @@ static __inline int rep_clz(rep_t a) { // many 64-bit platforms have this operation, but they tend to have hardware // floating-point, so we don't bother with a special case for them here. static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - // Each of the component 32x32 -> 64 products - const uint64_t plolo = loWord(a) * loWord(b); - const uint64_t plohi = loWord(a) * hiWord(b); - const uint64_t philo = hiWord(a) * loWord(b); - const uint64_t phihi = hiWord(a) * hiWord(b); - // Sum terms that contribute to lo in a way that allows us to get the carry - const uint64_t r0 = loWord(plolo); - const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); - *lo = r0 + (r1 << 32); - // Sum terms contributing to hi with the carry from lo - *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; + // Each of the component 32x32 -> 64 products + const uint64_t plolo = loWord(a) * loWord(b); + const uint64_t plohi = loWord(a) * hiWord(b); + const uint64_t philo = hiWord(a) * loWord(b); + const uint64_t phihi = hiWord(a) * hiWord(b); + // Sum terms that contribute to lo in a way that allows us to get the carry + const uint64_t r0 = loWord(plolo); + const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); + *lo = r0 + (r1 << 32); + // Sum terms contributing to hi with the carry from lo + *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; } #undef loWord #undef hiWord @@ -103,7 +100,7 @@ static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b); #elif defined QUAD_PRECISION -#if __LDBL_MANT_DIG__ == 113 +#if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__) #define CRT_LDBL_128BIT typedef __uint128_t rep_t; typedef __int128_t srep_t; @@ -114,32 +111,34 @@ typedef long double fp_t; #define significandBits 112 static __inline int rep_clz(rep_t a) { - const union - { - __uint128_t ll; + const union { + __uint128_t ll; #if _YUGA_BIG_ENDIAN - struct { uint64_t high, low; } s; + struct { + uint64_t high, low; + } s; #else - struct { uint64_t low, high; } s; + struct { + uint64_t low, high; + } s; #endif - } uu = { .ll = a }; + } uu = {.ll = a}; - uint64_t word; - uint64_t add; + uint64_t word; + uint64_t add; - if (uu.s.high){ - word = uu.s.high; - add = 0; - } - else{ - word = uu.s.low; - add = 64; - } - return __builtin_clzll(word) + add; + if (uu.s.high) { + word = uu.s.high; + add = 0; + } else { + word = uu.s.low; + add = 64; + } + return __builtin_clzll(word) + add; } -#define Word_LoMask UINT64_C(0x00000000ffffffff) -#define Word_HiMask UINT64_C(0xffffffff00000000) +#define Word_LoMask UINT64_C(0x00000000ffffffff) +#define Word_HiMask UINT64_C(0xffffffff00000000) #define Word_FullMask UINT64_C(0xffffffffffffffff) #define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask) #define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask) @@ -151,55 +150,41 @@ static __inline int rep_clz(rep_t a) { // floating-point, so we don't bother with a special case for them here. static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - const uint64_t product11 = Word_1(a) * Word_1(b); - const uint64_t product12 = Word_1(a) * Word_2(b); - const uint64_t product13 = Word_1(a) * Word_3(b); - const uint64_t product14 = Word_1(a) * Word_4(b); - const uint64_t product21 = Word_2(a) * Word_1(b); - const uint64_t product22 = Word_2(a) * Word_2(b); - const uint64_t product23 = Word_2(a) * Word_3(b); - const uint64_t product24 = Word_2(a) * Word_4(b); - const uint64_t product31 = Word_3(a) * Word_1(b); - const uint64_t product32 = Word_3(a) * Word_2(b); - const uint64_t product33 = Word_3(a) * Word_3(b); - const uint64_t product34 = Word_3(a) * Word_4(b); - const uint64_t product41 = Word_4(a) * Word_1(b); - const uint64_t product42 = Word_4(a) * Word_2(b); - const uint64_t product43 = Word_4(a) * Word_3(b); - const uint64_t product44 = Word_4(a) * Word_4(b); - - const __uint128_t sum0 = (__uint128_t)product44; - const __uint128_t sum1 = (__uint128_t)product34 + - (__uint128_t)product43; - const __uint128_t sum2 = (__uint128_t)product24 + - (__uint128_t)product33 + - (__uint128_t)product42; - const __uint128_t sum3 = (__uint128_t)product14 + - (__uint128_t)product23 + - (__uint128_t)product32 + - (__uint128_t)product41; - const __uint128_t sum4 = (__uint128_t)product13 + - (__uint128_t)product22 + - (__uint128_t)product31; - const __uint128_t sum5 = (__uint128_t)product12 + - (__uint128_t)product21; - const __uint128_t sum6 = (__uint128_t)product11; - - const __uint128_t r0 = (sum0 & Word_FullMask) + - ((sum1 & Word_LoMask) << 32); - const __uint128_t r1 = (sum0 >> 64) + - ((sum1 >> 32) & Word_FullMask) + - (sum2 & Word_FullMask) + - ((sum3 << 32) & Word_HiMask); - - *lo = r0 + (r1 << 64); - *hi = (r1 >> 64) + - (sum1 >> 96) + - (sum2 >> 64) + - (sum3 >> 32) + - sum4 + - (sum5 << 32) + - (sum6 << 64); + const uint64_t product11 = Word_1(a) * Word_1(b); + const uint64_t product12 = Word_1(a) * Word_2(b); + const uint64_t product13 = Word_1(a) * Word_3(b); + const uint64_t product14 = Word_1(a) * Word_4(b); + const uint64_t product21 = Word_2(a) * Word_1(b); + const uint64_t product22 = Word_2(a) * Word_2(b); + const uint64_t product23 = Word_2(a) * Word_3(b); + const uint64_t product24 = Word_2(a) * Word_4(b); + const uint64_t product31 = Word_3(a) * Word_1(b); + const uint64_t product32 = Word_3(a) * Word_2(b); + const uint64_t product33 = Word_3(a) * Word_3(b); + const uint64_t product34 = Word_3(a) * Word_4(b); + const uint64_t product41 = Word_4(a) * Word_1(b); + const uint64_t product42 = Word_4(a) * Word_2(b); + const uint64_t product43 = Word_4(a) * Word_3(b); + const uint64_t product44 = Word_4(a) * Word_4(b); + + const __uint128_t sum0 = (__uint128_t)product44; + const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43; + const __uint128_t sum2 = + (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42; + const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 + + (__uint128_t)product32 + (__uint128_t)product41; + const __uint128_t sum4 = + (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31; + const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21; + const __uint128_t sum6 = (__uint128_t)product11; + + const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32); + const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) + + (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask); + + *lo = r0 + (r1 << 64); + *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 + + (sum5 << 32) + (sum6 << 64); } #undef Word_1 #undef Word_2 @@ -208,63 +193,70 @@ static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { #undef Word_HiMask #undef Word_LoMask #undef Word_FullMask -#endif // __LDBL_MANT_DIG__ == 113 +#endif // __LDBL_MANT_DIG__ == 113 && __SIZEOF_INT128__ #else #error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined. #endif -#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || defined(CRT_LDBL_128BIT) -#define typeWidth (sizeof(rep_t)*CHAR_BIT) -#define exponentBits (typeWidth - significandBits - 1) -#define maxExponent ((1 << exponentBits) - 1) -#define exponentBias (maxExponent >> 1) +#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || \ + defined(CRT_LDBL_128BIT) +#define typeWidth (sizeof(rep_t) * CHAR_BIT) +#define exponentBits (typeWidth - significandBits - 1) +#define maxExponent ((1 << exponentBits) - 1) +#define exponentBias (maxExponent >> 1) -#define implicitBit (REP_C(1) << significandBits) +#define implicitBit (REP_C(1) << significandBits) #define significandMask (implicitBit - 1U) -#define signBit (REP_C(1) << (significandBits + exponentBits)) -#define absMask (signBit - 1U) -#define exponentMask (absMask ^ significandMask) -#define oneRep ((rep_t)exponentBias << significandBits) -#define infRep exponentMask -#define quietBit (implicitBit >> 1) -#define qnanRep (exponentMask | quietBit) +#define signBit (REP_C(1) << (significandBits + exponentBits)) +#define absMask (signBit - 1U) +#define exponentMask (absMask ^ significandMask) +#define oneRep ((rep_t)exponentBias << significandBits) +#define infRep exponentMask +#define quietBit (implicitBit >> 1) +#define qnanRep (exponentMask | quietBit) static __inline rep_t toRep(fp_t x) { - const union { fp_t f; rep_t i; } rep = {.f = x}; - return rep.i; + const union { + fp_t f; + rep_t i; + } rep = {.f = x}; + return rep.i; } static __inline fp_t fromRep(rep_t x) { - const union { fp_t f; rep_t i; } rep = {.i = x}; - return rep.f; + const union { + fp_t f; + rep_t i; + } rep = {.i = x}; + return rep.f; } static __inline int normalize(rep_t *significand) { - const int shift = rep_clz(*significand) - rep_clz(implicitBit); - *significand <<= shift; - return 1 - shift; + const int shift = rep_clz(*significand) - rep_clz(implicitBit); + *significand <<= shift; + return 1 - shift; } static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { - *hi = *hi << count | *lo >> (typeWidth - count); - *lo = *lo << count; + *hi = *hi << count | *lo >> (typeWidth - count); + *lo = *lo << count; } -static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, unsigned int count) { - if (count < typeWidth) { - const bool sticky = *lo << (typeWidth - count); - *lo = *hi << (typeWidth - count) | *lo >> count | sticky; - *hi = *hi >> count; - } - else if (count < 2*typeWidth) { - const bool sticky = *hi << (2*typeWidth - count) | *lo; - *lo = *hi >> (count - typeWidth) | sticky; - *hi = 0; - } else { - const bool sticky = *hi | *lo; - *lo = sticky; - *hi = 0; - } +static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, + unsigned int count) { + if (count < typeWidth) { + const bool sticky = *lo << (typeWidth - count); + *lo = *hi << (typeWidth - count) | *lo >> count | sticky; + *hi = *hi >> count; + } else if (count < 2 * typeWidth) { + const bool sticky = *hi << (2 * typeWidth - count) | *lo; + *lo = *hi >> (count - typeWidth) | sticky; + *hi = 0; + } else { + const bool sticky = *hi | *lo; + *lo = sticky; + *hi = 0; + } } // Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids @@ -280,9 +272,9 @@ static __inline fp_t __compiler_rt_logbX(fp_t x) { // 2) 0.0 returns -inf if (exp == maxExponent) { if (((rep & signBit) == 0) || (x != x)) { - return x; // NaN or +inf: return x + return x; // NaN or +inf: return x } else { - return -x; // -inf: return -x + return -x; // -inf: return -x } } else if (x == 0.0) { // 0.0: return -inf @@ -291,13 +283,13 @@ static __inline fp_t __compiler_rt_logbX(fp_t x) { if (exp != 0) { // Normal number - return exp - exponentBias; // Unbias exponent + return exp - exponentBias; // Unbias exponent } else { // Subnormal number; normalize and repeat rep &= absMask; const int shift = 1 - normalize(&rep); exp = (rep & exponentMask) >> significandBits; - return exp - exponentBias - shift; // Unbias exponent + return exp - exponentBias - shift; // Unbias exponent } } #endif @@ -311,17 +303,17 @@ static __inline fp_t __compiler_rt_logb(fp_t x) { return __compiler_rt_logbX(x); } #elif defined(QUAD_PRECISION) - #if defined(CRT_LDBL_128BIT) +#if defined(CRT_LDBL_128BIT) static __inline fp_t __compiler_rt_logbl(fp_t x) { return __compiler_rt_logbX(x); } - #else +#else // The generic implementation only works for ieee754 floating point. For other // floating point types, continue to rely on the libm implementation for now. static __inline long double __compiler_rt_logbl(long double x) { return crt_logbl(x); } - #endif +#endif #endif #endif // FP_LIB_HEADER diff --git a/lib/builtins/fp_mul_impl.inc b/lib/builtins/fp_mul_impl.inc index b34aa1b8f544..a93f2d78ad6b 100644 --- a/lib/builtins/fp_mul_impl.inc +++ b/lib/builtins/fp_mul_impl.inc @@ -1,9 +1,8 @@ //===---- lib/fp_mul_impl.inc - floating point multiplication -----*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,102 +14,115 @@ #include "fp_lib.h" static __inline fp_t __mulXf3__(fp_t a, fp_t b) { - const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; - const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; - const rep_t productSign = (toRep(a) ^ toRep(b)) & signBit; - - rep_t aSignificand = toRep(a) & significandMask; - rep_t bSignificand = toRep(b) & significandMask; - int scale = 0; - - // Detect if a or b is zero, denormal, infinity, or NaN. - if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - - const rep_t aAbs = toRep(a) & absMask; - const rep_t bAbs = toRep(b) & absMask; - - // NaN * anything = qNaN - if (aAbs > infRep) return fromRep(toRep(a) | quietBit); - // anything * NaN = qNaN - if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - - if (aAbs == infRep) { - // infinity * non-zero = +/- infinity - if (bAbs) return fromRep(aAbs | productSign); - // infinity * zero = NaN - else return fromRep(qnanRep); - } - - if (bAbs == infRep) { - //? non-zero * infinity = +/- infinity - if (aAbs) return fromRep(bAbs | productSign); - // zero * infinity = NaN - else return fromRep(qnanRep); - } - - // zero * anything = +/- zero - if (!aAbs) return fromRep(productSign); - // anything * zero = +/- zero - if (!bAbs) return fromRep(productSign); - - // one or both of a or b is denormal, the other (if applicable) is a - // normal number. Renormalize one or both of a and b, and set scale to - // include the necessary exponent adjustment. - if (aAbs < implicitBit) scale += normalize(&aSignificand); - if (bAbs < implicitBit) scale += normalize(&bSignificand); + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; + const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; + const rep_t productSign = (toRep(a) ^ toRep(b)) & signBit; + + rep_t aSignificand = toRep(a) & significandMask; + rep_t bSignificand = toRep(b) & significandMask; + int scale = 0; + + // Detect if a or b is zero, denormal, infinity, or NaN. + if (aExponent - 1U >= maxExponent - 1U || + bExponent - 1U >= maxExponent - 1U) { + + const rep_t aAbs = toRep(a) & absMask; + const rep_t bAbs = toRep(b) & absMask; + + // NaN * anything = qNaN + if (aAbs > infRep) + return fromRep(toRep(a) | quietBit); + // anything * NaN = qNaN + if (bAbs > infRep) + return fromRep(toRep(b) | quietBit); + + if (aAbs == infRep) { + // infinity * non-zero = +/- infinity + if (bAbs) + return fromRep(aAbs | productSign); + // infinity * zero = NaN + else + return fromRep(qnanRep); } - // Or in the implicit significand bit. (If we fell through from the - // denormal path it was already set by normalize( ), but setting it twice - // won't hurt anything.) - aSignificand |= implicitBit; - bSignificand |= implicitBit; - - // Get the significand of a*b. Before multiplying the significands, shift - // one of them left to left-align it in the field. Thus, the product will - // have (exponentBits + 2) integral digits, all but two of which must be - // zero. Normalizing this result is just a conditional left-shift by one - // and bumping the exponent accordingly. - rep_t productHi, productLo; - wideMultiply(aSignificand, bSignificand << exponentBits, - &productHi, &productLo); - - int productExponent = aExponent + bExponent - exponentBias + scale; - - // Normalize the significand, adjust exponent if needed. - if (productHi & implicitBit) productExponent++; - else wideLeftShift(&productHi, &productLo, 1); - - // If we have overflowed the type, return +/- infinity. - if (productExponent >= maxExponent) return fromRep(infRep | productSign); - - if (productExponent <= 0) { - // Result is denormal before rounding - // - // If the result is so small that it just underflows to zero, return - // a zero of the appropriate sign. Mathematically there is no need to - // handle this case separately, but we make it a special case to - // simplify the shift logic. - const unsigned int shift = REP_C(1) - (unsigned int)productExponent; - if (shift >= typeWidth) return fromRep(productSign); - - // Otherwise, shift the significand of the result so that the round - // bit is the high bit of productLo. - wideRightShiftWithSticky(&productHi, &productLo, shift); - } - else { - // Result is normal before rounding; insert the exponent. - productHi &= significandMask; - productHi |= (rep_t)productExponent << significandBits; + if (bAbs == infRep) { + // non-zero * infinity = +/- infinity + if (aAbs) + return fromRep(bAbs | productSign); + // zero * infinity = NaN + else + return fromRep(qnanRep); } - // Insert the sign of the result: - productHi |= productSign; - - // Final rounding. The final result may overflow to infinity, or underflow - // to zero, but those are the correct results in those cases. We use the - // default IEEE-754 round-to-nearest, ties-to-even rounding mode. - if (productLo > signBit) productHi++; - if (productLo == signBit) productHi += productHi & 1; - return fromRep(productHi); + // zero * anything = +/- zero + if (!aAbs) + return fromRep(productSign); + // anything * zero = +/- zero + if (!bAbs) + return fromRep(productSign); + + // One or both of a or b is denormal. The other (if applicable) is a + // normal number. Renormalize one or both of a and b, and set scale to + // include the necessary exponent adjustment. + if (aAbs < implicitBit) + scale += normalize(&aSignificand); + if (bAbs < implicitBit) + scale += normalize(&bSignificand); + } + + // Set the implicit significand bit. If we fell through from the + // denormal path it was already set by normalize( ), but setting it twice + // won't hurt anything. + aSignificand |= implicitBit; + bSignificand |= implicitBit; + + // Perform a basic multiplication on the significands. One of them must be + // shifted beforehand to be aligned with the exponent. + rep_t productHi, productLo; + wideMultiply(aSignificand, bSignificand << exponentBits, &productHi, + &productLo); + + int productExponent = aExponent + bExponent - exponentBias + scale; + + // Normalize the significand and adjust the exponent if needed. + if (productHi & implicitBit) + productExponent++; + else + wideLeftShift(&productHi, &productLo, 1); + + // If we have overflowed the type, return +/- infinity. + if (productExponent >= maxExponent) + return fromRep(infRep | productSign); + + if (productExponent <= 0) { + // The result is denormal before rounding. + // + // If the result is so small that it just underflows to zero, return + // zero with the appropriate sign. Mathematically, there is no need to + // handle this case separately, but we make it a special case to + // simplify the shift logic. + const unsigned int shift = REP_C(1) - (unsigned int)productExponent; + if (shift >= typeWidth) + return fromRep(productSign); + + // Otherwise, shift the significand of the result so that the round + // bit is the high bit of productLo. + wideRightShiftWithSticky(&productHi, &productLo, shift); + } else { + // The result is normal before rounding. Insert the exponent. + productHi &= significandMask; + productHi |= (rep_t)productExponent << significandBits; + } + + // Insert the sign of the result. + productHi |= productSign; + + // Perform the final rounding. The final result may overflow to infinity, + // or underflow to zero, but those are the correct results in those cases. + // We use the default IEEE-754 round-to-nearest, ties-to-even rounding mode. + if (productLo > signBit) + productHi++; + if (productLo == signBit) + productHi += productHi & 1; + return fromRep(productHi); } diff --git a/lib/builtins/fp_trunc.h b/lib/builtins/fp_trunc.h index d5e79bb5b863..aca4c9b6e677 100644 --- a/lib/builtins/fp_trunc.h +++ b/lib/builtins/fp_trunc.h @@ -1,9 +1,8 @@ //=== lib/fp_trunc.h - high precision -> low precision conversion *- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,7 +35,7 @@ static const int srcSigBits = 112; #else #error Source should be double precision or quad precision! -#endif //end source precision +#endif // end source precision #if defined DST_DOUBLE typedef double dst_t; @@ -58,19 +57,25 @@ static const int dstSigBits = 10; #else #error Destination should be single precision or double precision! -#endif //end destination precision +#endif // end destination precision // End of specialization parameters. Two helper routines for conversion to and // from the representation of floating-point data as integer values follow. static __inline src_rep_t srcToRep(src_t x) { - const union { src_t f; src_rep_t i; } rep = {.f = x}; - return rep.i; + const union { + src_t f; + src_rep_t i; + } rep = {.f = x}; + return rep.i; } static __inline dst_t dstFromRep(dst_rep_t x) { - const union { dst_t f; dst_rep_t i; } rep = {.i = x}; - return rep.f; + const union { + dst_t f; + dst_rep_t i; + } rep = {.i = x}; + return rep.f; } #endif // FP_TRUNC_HEADER diff --git a/lib/builtins/fp_trunc_impl.inc b/lib/builtins/fp_trunc_impl.inc index d88ae060913f..133c8bbe5c2f 100644 --- a/lib/builtins/fp_trunc_impl.inc +++ b/lib/builtins/fp_trunc_impl.inc @@ -1,9 +1,8 @@ //= lib/fp_trunc_impl.inc - high precision -> low precision conversion *-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,107 +28,105 @@ // // Finally, the following assumptions are made: // -// 1. floating-point types and integer types have the same endianness on the -// target platform +// 1. Floating-point types and integer types have the same endianness on the +// target platform. // -// 2. quiet NaNs, if supported, are indicated by the leading bit of the -// significand field being set +// 2. Quiet NaNs, if supported, are indicated by the leading bit of the +// significand field being set. // //===----------------------------------------------------------------------===// #include "fp_trunc.h" static __inline dst_t __truncXfYf2__(src_t a) { - // Various constants whose values follow from the type parameters. - // Any reasonable optimizer will fold and propagate all of these. - const int srcBits = sizeof(src_t)*CHAR_BIT; - const int srcExpBits = srcBits - srcSigBits - 1; - const int srcInfExp = (1 << srcExpBits) - 1; - const int srcExpBias = srcInfExp >> 1; + // Various constants whose values follow from the type parameters. + // Any reasonable optimizer will fold and propagate all of these. + const int srcBits = sizeof(src_t) * CHAR_BIT; + const int srcExpBits = srcBits - srcSigBits - 1; + const int srcInfExp = (1 << srcExpBits) - 1; + const int srcExpBias = srcInfExp >> 1; - const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; - const src_rep_t srcSignificandMask = srcMinNormal - 1; - const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; - const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); - const src_rep_t srcAbsMask = srcSignMask - 1; - const src_rep_t roundMask = (SRC_REP_C(1) << (srcSigBits - dstSigBits)) - 1; - const src_rep_t halfway = SRC_REP_C(1) << (srcSigBits - dstSigBits - 1); - const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); - const src_rep_t srcNaNCode = srcQNaN - 1; + const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits; + const src_rep_t srcSignificandMask = srcMinNormal - 1; + const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits; + const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits); + const src_rep_t srcAbsMask = srcSignMask - 1; + const src_rep_t roundMask = (SRC_REP_C(1) << (srcSigBits - dstSigBits)) - 1; + const src_rep_t halfway = SRC_REP_C(1) << (srcSigBits - dstSigBits - 1); + const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1); + const src_rep_t srcNaNCode = srcQNaN - 1; - const int dstBits = sizeof(dst_t)*CHAR_BIT; - const int dstExpBits = dstBits - dstSigBits - 1; - const int dstInfExp = (1 << dstExpBits) - 1; - const int dstExpBias = dstInfExp >> 1; + const int dstBits = sizeof(dst_t) * CHAR_BIT; + const int dstExpBits = dstBits - dstSigBits - 1; + const int dstInfExp = (1 << dstExpBits) - 1; + const int dstExpBias = dstInfExp >> 1; - const int underflowExponent = srcExpBias + 1 - dstExpBias; - const int overflowExponent = srcExpBias + dstInfExp - dstExpBias; - const src_rep_t underflow = (src_rep_t)underflowExponent << srcSigBits; - const src_rep_t overflow = (src_rep_t)overflowExponent << srcSigBits; + const int underflowExponent = srcExpBias + 1 - dstExpBias; + const int overflowExponent = srcExpBias + dstInfExp - dstExpBias; + const src_rep_t underflow = (src_rep_t)underflowExponent << srcSigBits; + const src_rep_t overflow = (src_rep_t)overflowExponent << srcSigBits; - const dst_rep_t dstQNaN = DST_REP_C(1) << (dstSigBits - 1); - const dst_rep_t dstNaNCode = dstQNaN - 1; + const dst_rep_t dstQNaN = DST_REP_C(1) << (dstSigBits - 1); + const dst_rep_t dstNaNCode = dstQNaN - 1; - // Break a into a sign and representation of the absolute value - const src_rep_t aRep = srcToRep(a); - const src_rep_t aAbs = aRep & srcAbsMask; - const src_rep_t sign = aRep & srcSignMask; - dst_rep_t absResult; + // Break a into a sign and representation of the absolute value. + const src_rep_t aRep = srcToRep(a); + const src_rep_t aAbs = aRep & srcAbsMask; + const src_rep_t sign = aRep & srcSignMask; + dst_rep_t absResult; - if (aAbs - underflow < aAbs - overflow) { - // The exponent of a is within the range of normal numbers in the - // destination format. We can convert by simply right-shifting with - // rounding and adjusting the exponent. - absResult = aAbs >> (srcSigBits - dstSigBits); - absResult -= (dst_rep_t)(srcExpBias - dstExpBias) << dstSigBits; + if (aAbs - underflow < aAbs - overflow) { + // The exponent of a is within the range of normal numbers in the + // destination format. We can convert by simply right-shifting with + // rounding and adjusting the exponent. + absResult = aAbs >> (srcSigBits - dstSigBits); + absResult -= (dst_rep_t)(srcExpBias - dstExpBias) << dstSigBits; - const src_rep_t roundBits = aAbs & roundMask; - // Round to nearest - if (roundBits > halfway) - absResult++; - // Ties to even - else if (roundBits == halfway) - absResult += absResult & 1; - } - else if (aAbs > srcInfinity) { - // a is NaN. - // Conjure the result by beginning with infinity, setting the qNaN - // bit and inserting the (truncated) trailing NaN field. - absResult = (dst_rep_t)dstInfExp << dstSigBits; - absResult |= dstQNaN; - absResult |= ((aAbs & srcNaNCode) >> (srcSigBits - dstSigBits)) & dstNaNCode; - } - else if (aAbs >= overflow) { - // a overflows to infinity. - absResult = (dst_rep_t)dstInfExp << dstSigBits; - } - else { - // a underflows on conversion to the destination type or is an exact - // zero. The result may be a denormal or zero. Extract the exponent - // to get the shift amount for the denormalization. - const int aExp = aAbs >> srcSigBits; - const int shift = srcExpBias - dstExpBias - aExp + 1; + const src_rep_t roundBits = aAbs & roundMask; + // Round to nearest. + if (roundBits > halfway) + absResult++; + // Tie to even. + else if (roundBits == halfway) + absResult += absResult & 1; + } else if (aAbs > srcInfinity) { + // a is NaN. + // Conjure the result by beginning with infinity, setting the qNaN + // bit and inserting the (truncated) trailing NaN field. + absResult = (dst_rep_t)dstInfExp << dstSigBits; + absResult |= dstQNaN; + absResult |= + ((aAbs & srcNaNCode) >> (srcSigBits - dstSigBits)) & dstNaNCode; + } else if (aAbs >= overflow) { + // a overflows to infinity. + absResult = (dst_rep_t)dstInfExp << dstSigBits; + } else { + // a underflows on conversion to the destination type or is an exact + // zero. The result may be a denormal or zero. Extract the exponent + // to get the shift amount for the denormalization. + const int aExp = aAbs >> srcSigBits; + const int shift = srcExpBias - dstExpBias - aExp + 1; - const src_rep_t significand = (aRep & srcSignificandMask) | srcMinNormal; + const src_rep_t significand = (aRep & srcSignificandMask) | srcMinNormal; - // Right shift by the denormalization amount with sticky. - if (shift > srcSigBits) { - absResult = 0; - } else { - const bool sticky = significand << (srcBits - shift); - src_rep_t denormalizedSignificand = significand >> shift | sticky; - absResult = denormalizedSignificand >> (srcSigBits - dstSigBits); - const src_rep_t roundBits = denormalizedSignificand & roundMask; - // Round to nearest - if (roundBits > halfway) - absResult++; - // Ties to even - else if (roundBits == halfway) - absResult += absResult & 1; - } + // Right shift by the denormalization amount with sticky. + if (shift > srcSigBits) { + absResult = 0; + } else { + const bool sticky = significand << (srcBits - shift); + src_rep_t denormalizedSignificand = significand >> shift | sticky; + absResult = denormalizedSignificand >> (srcSigBits - dstSigBits); + const src_rep_t roundBits = denormalizedSignificand & roundMask; + // Round to nearest + if (roundBits > halfway) + absResult++; + // Ties to even + else if (roundBits == halfway) + absResult += absResult & 1; } + } - // Apply the signbit to (dst_t)abs(a). - const dst_rep_t result = absResult | sign >> (srcBits - dstBits); - return dstFromRep(result); + // Apply the signbit to the absolute value. + const dst_rep_t result = absResult | sign >> (srcBits - dstBits); + return dstFromRep(result); } diff --git a/lib/builtins/gcc_personality_v0.c b/lib/builtins/gcc_personality_v0.c index 68581ef16434..d12ee03c49fe 100644 --- a/lib/builtins/gcc_personality_v0.c +++ b/lib/builtins/gcc_personality_v0.c @@ -1,145 +1,135 @@ -/* ===-- gcc_personality_v0.c - Implement __gcc_personality_v0 -------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - */ +//===-- gcc_personality_v0.c - Implement __gcc_personality_v0 -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include <unwind.h> -#if defined(__arm__) && !defined(__ARM_DWARF_EH__) && !defined(__USING_SJLJ_EXCEPTIONS__) -/* - * When building with older compilers (e.g. clang <3.9), it is possible that we - * have a version of unwind.h which does not provide the EHABI declarations - * which are quired for the C personality to conform to the specification. In - * order to provide forward compatibility for such compilers, we re-declare the - * necessary interfaces in the helper to permit a standalone compilation of the - * builtins (which contains the C unwinding personality for historical reasons). - */ +#if defined(__arm__) && !defined(__ARM_DWARF_EH__) && \ + !defined(__USING_SJLJ_EXCEPTIONS__) +// When building with older compilers (e.g. clang <3.9), it is possible that we +// have a version of unwind.h which does not provide the EHABI declarations +// which are quired for the C personality to conform to the specification. In +// order to provide forward compatibility for such compilers, we re-declare the +// necessary interfaces in the helper to permit a standalone compilation of the +// builtins (which contains the C unwinding personality for historical reasons). #include "unwind-ehabi-helpers.h" #endif -/* - * Pointer encodings documented at: - * http://refspecs.freestandards.org/LSB_1.3.0/gLSB/gLSB/ehframehdr.html - */ - -#define DW_EH_PE_omit 0xff /* no data follows */ - -#define DW_EH_PE_absptr 0x00 -#define DW_EH_PE_uleb128 0x01 -#define DW_EH_PE_udata2 0x02 -#define DW_EH_PE_udata4 0x03 -#define DW_EH_PE_udata8 0x04 -#define DW_EH_PE_sleb128 0x09 -#define DW_EH_PE_sdata2 0x0A -#define DW_EH_PE_sdata4 0x0B -#define DW_EH_PE_sdata8 0x0C - -#define DW_EH_PE_pcrel 0x10 -#define DW_EH_PE_textrel 0x20 -#define DW_EH_PE_datarel 0x30 -#define DW_EH_PE_funcrel 0x40 -#define DW_EH_PE_aligned 0x50 -#define DW_EH_PE_indirect 0x80 /* gcc extension */ - - - -/* read a uleb128 encoded value and advance pointer */ -static uintptr_t readULEB128(const uint8_t** data) -{ - uintptr_t result = 0; - uintptr_t shift = 0; - unsigned char byte; - const uint8_t* p = *data; - do { - byte = *p++; - result |= (byte & 0x7f) << shift; - shift += 7; - } while (byte & 0x80); - *data = p; - return result; +// Pointer encodings documented at: +// http://refspecs.freestandards.org/LSB_1.3.0/gLSB/gLSB/ehframehdr.html + +#define DW_EH_PE_omit 0xff // no data follows + +#define DW_EH_PE_absptr 0x00 +#define DW_EH_PE_uleb128 0x01 +#define DW_EH_PE_udata2 0x02 +#define DW_EH_PE_udata4 0x03 +#define DW_EH_PE_udata8 0x04 +#define DW_EH_PE_sleb128 0x09 +#define DW_EH_PE_sdata2 0x0A +#define DW_EH_PE_sdata4 0x0B +#define DW_EH_PE_sdata8 0x0C + +#define DW_EH_PE_pcrel 0x10 +#define DW_EH_PE_textrel 0x20 +#define DW_EH_PE_datarel 0x30 +#define DW_EH_PE_funcrel 0x40 +#define DW_EH_PE_aligned 0x50 +#define DW_EH_PE_indirect 0x80 // gcc extension + +// read a uleb128 encoded value and advance pointer +static uintptr_t readULEB128(const uint8_t **data) { + uintptr_t result = 0; + uintptr_t shift = 0; + unsigned char byte; + const uint8_t *p = *data; + do { + byte = *p++; + result |= (byte & 0x7f) << shift; + shift += 7; + } while (byte & 0x80); + *data = p; + return result; } -/* read a pointer encoded value and advance pointer */ -static uintptr_t readEncodedPointer(const uint8_t** data, uint8_t encoding) -{ - const uint8_t* p = *data; - uintptr_t result = 0; - - if ( encoding == DW_EH_PE_omit ) - return 0; - - /* first get value */ - switch (encoding & 0x0F) { - case DW_EH_PE_absptr: - result = *((const uintptr_t*)p); - p += sizeof(uintptr_t); - break; - case DW_EH_PE_uleb128: - result = readULEB128(&p); - break; - case DW_EH_PE_udata2: - result = *((const uint16_t*)p); - p += sizeof(uint16_t); - break; - case DW_EH_PE_udata4: - result = *((const uint32_t*)p); - p += sizeof(uint32_t); - break; - case DW_EH_PE_udata8: - result = *((const uint64_t*)p); - p += sizeof(uint64_t); - break; - case DW_EH_PE_sdata2: - result = *((const int16_t*)p); - p += sizeof(int16_t); - break; - case DW_EH_PE_sdata4: - result = *((const int32_t*)p); - p += sizeof(int32_t); - break; - case DW_EH_PE_sdata8: - result = *((const int64_t*)p); - p += sizeof(int64_t); - break; - case DW_EH_PE_sleb128: - default: - /* not supported */ - compilerrt_abort(); - break; - } - - /* then add relative offset */ - switch ( encoding & 0x70 ) { - case DW_EH_PE_absptr: - /* do nothing */ - break; - case DW_EH_PE_pcrel: - result += (uintptr_t)(*data); - break; - case DW_EH_PE_textrel: - case DW_EH_PE_datarel: - case DW_EH_PE_funcrel: - case DW_EH_PE_aligned: - default: - /* not supported */ - compilerrt_abort(); - break; - } - - /* then apply indirection */ - if (encoding & DW_EH_PE_indirect) { - result = *((const uintptr_t*)result); - } - - *data = p; - return result; +// read a pointer encoded value and advance pointer +static uintptr_t readEncodedPointer(const uint8_t **data, uint8_t encoding) { + const uint8_t *p = *data; + uintptr_t result = 0; + + if (encoding == DW_EH_PE_omit) + return 0; + + // first get value + switch (encoding & 0x0F) { + case DW_EH_PE_absptr: + result = *((const uintptr_t *)p); + p += sizeof(uintptr_t); + break; + case DW_EH_PE_uleb128: + result = readULEB128(&p); + break; + case DW_EH_PE_udata2: + result = *((const uint16_t *)p); + p += sizeof(uint16_t); + break; + case DW_EH_PE_udata4: + result = *((const uint32_t *)p); + p += sizeof(uint32_t); + break; + case DW_EH_PE_udata8: + result = *((const uint64_t *)p); + p += sizeof(uint64_t); + break; + case DW_EH_PE_sdata2: + result = *((const int16_t *)p); + p += sizeof(int16_t); + break; + case DW_EH_PE_sdata4: + result = *((const int32_t *)p); + p += sizeof(int32_t); + break; + case DW_EH_PE_sdata8: + result = *((const int64_t *)p); + p += sizeof(int64_t); + break; + case DW_EH_PE_sleb128: + default: + // not supported + compilerrt_abort(); + break; + } + + // then add relative offset + switch (encoding & 0x70) { + case DW_EH_PE_absptr: + // do nothing + break; + case DW_EH_PE_pcrel: + result += (uintptr_t)(*data); + break; + case DW_EH_PE_textrel: + case DW_EH_PE_datarel: + case DW_EH_PE_funcrel: + case DW_EH_PE_aligned: + default: + // not supported + compilerrt_abort(); + break; + } + + // then apply indirection + if (encoding & DW_EH_PE_indirect) { + result = *((const uintptr_t *)result); + } + + *data = p; + return result; } #if defined(__arm__) && !defined(__USING_SJLJ_EXCEPTIONS__) && \ @@ -153,99 +143,92 @@ static inline _Unwind_Reason_Code continueUnwind(struct _Unwind_Exception *exceptionObject, struct _Unwind_Context *context) { #if USING_ARM_EHABI - /* - * On ARM EHABI the personality routine is responsible for actually - * unwinding a single stack frame before returning (ARM EHABI Sec. 6.1). - */ - if (__gnu_unwind_frame(exceptionObject, context) != _URC_OK) - return _URC_FAILURE; + // On ARM EHABI the personality routine is responsible for actually + // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1). + if (__gnu_unwind_frame(exceptionObject, context) != _URC_OK) + return _URC_FAILURE; #endif - return _URC_CONTINUE_UNWIND; + return _URC_CONTINUE_UNWIND; } -/* - * The C compiler makes references to __gcc_personality_v0 in - * the dwarf unwind information for translation units that use - * __attribute__((cleanup(xx))) on local variables. - * This personality routine is called by the system unwinder - * on each frame as the stack is unwound during a C++ exception - * throw through a C function compiled with -fexceptions. - */ +// The C compiler makes references to __gcc_personality_v0 in +// the dwarf unwind information for translation units that use +// __attribute__((cleanup(xx))) on local variables. +// This personality routine is called by the system unwinder +// on each frame as the stack is unwound during a C++ exception +// throw through a C function compiled with -fexceptions. #if __USING_SJLJ_EXCEPTIONS__ -/* the setjump-longjump based exceptions personality routine has a - * different name */ -COMPILER_RT_ABI _Unwind_Reason_Code -__gcc_personality_sj0(int version, _Unwind_Action actions, - uint64_t exceptionClass, struct _Unwind_Exception* exceptionObject, - struct _Unwind_Context *context) +// the setjump-longjump based exceptions personality routine has a +// different name +COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_sj0( + int version, _Unwind_Action actions, uint64_t exceptionClass, + struct _Unwind_Exception *exceptionObject, struct _Unwind_Context *context) #elif USING_ARM_EHABI -/* The ARM EHABI personality routine has a different signature. */ +// The ARM EHABI personality routine has a different signature. COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( - _Unwind_State state, struct _Unwind_Exception *exceptionObject, - struct _Unwind_Context *context) + _Unwind_State state, struct _Unwind_Exception *exceptionObject, + struct _Unwind_Context *context) #else -COMPILER_RT_ABI _Unwind_Reason_Code -__gcc_personality_v0(int version, _Unwind_Action actions, - uint64_t exceptionClass, struct _Unwind_Exception* exceptionObject, - struct _Unwind_Context *context) +COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( + int version, _Unwind_Action actions, uint64_t exceptionClass, + struct _Unwind_Exception *exceptionObject, struct _Unwind_Context *context) #endif { - /* Since C does not have catch clauses, there is nothing to do during */ - /* phase 1 (the search phase). */ + // Since C does not have catch clauses, there is nothing to do during + // phase 1 (the search phase). #if USING_ARM_EHABI - /* After resuming from a cleanup we should also continue on to the next - * frame straight away. */ - if ((state & _US_ACTION_MASK) != _US_UNWIND_FRAME_STARTING) + // After resuming from a cleanup we should also continue on to the next + // frame straight away. + if ((state & _US_ACTION_MASK) != _US_UNWIND_FRAME_STARTING) #else - if ( actions & _UA_SEARCH_PHASE ) + if (actions & _UA_SEARCH_PHASE) #endif - return continueUnwind(exceptionObject, context); - - /* There is nothing to do if there is no LSDA for this frame. */ - const uint8_t* lsda = (uint8_t*)_Unwind_GetLanguageSpecificData(context); - if ( lsda == (uint8_t*) 0 ) - return continueUnwind(exceptionObject, context); + return continueUnwind(exceptionObject, context); - uintptr_t pc = (uintptr_t)_Unwind_GetIP(context)-1; - uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context); - uintptr_t pcOffset = pc - funcStart; + // There is nothing to do if there is no LSDA for this frame. + const uint8_t *lsda = (uint8_t *)_Unwind_GetLanguageSpecificData(context); + if (lsda == (uint8_t *)0) + return continueUnwind(exceptionObject, context); - /* Parse LSDA header. */ - uint8_t lpStartEncoding = *lsda++; - if (lpStartEncoding != DW_EH_PE_omit) { - readEncodedPointer(&lsda, lpStartEncoding); - } - uint8_t ttypeEncoding = *lsda++; - if (ttypeEncoding != DW_EH_PE_omit) { - readULEB128(&lsda); - } - /* Walk call-site table looking for range that includes current PC. */ - uint8_t callSiteEncoding = *lsda++; - uint32_t callSiteTableLength = readULEB128(&lsda); - const uint8_t* callSiteTableStart = lsda; - const uint8_t* callSiteTableEnd = callSiteTableStart + callSiteTableLength; - const uint8_t* p=callSiteTableStart; - while (p < callSiteTableEnd) { - uintptr_t start = readEncodedPointer(&p, callSiteEncoding); - uintptr_t length = readEncodedPointer(&p, callSiteEncoding); - uintptr_t landingPad = readEncodedPointer(&p, callSiteEncoding); - readULEB128(&p); /* action value not used for C code */ - if ( landingPad == 0 ) - continue; /* no landing pad for this entry */ - if ( (start <= pcOffset) && (pcOffset < (start+length)) ) { - /* Found landing pad for the PC. - * Set Instruction Pointer to so we re-enter function - * at landing pad. The landing pad is created by the compiler - * to take two parameters in registers. - */ - _Unwind_SetGR(context, __builtin_eh_return_data_regno(0), - (uintptr_t)exceptionObject); - _Unwind_SetGR(context, __builtin_eh_return_data_regno(1), 0); - _Unwind_SetIP(context, (funcStart + landingPad)); - return _URC_INSTALL_CONTEXT; - } + uintptr_t pc = (uintptr_t)_Unwind_GetIP(context) - 1; + uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context); + uintptr_t pcOffset = pc - funcStart; + + // Parse LSDA header. + uint8_t lpStartEncoding = *lsda++; + if (lpStartEncoding != DW_EH_PE_omit) { + readEncodedPointer(&lsda, lpStartEncoding); + } + uint8_t ttypeEncoding = *lsda++; + if (ttypeEncoding != DW_EH_PE_omit) { + readULEB128(&lsda); + } + // Walk call-site table looking for range that includes current PC. + uint8_t callSiteEncoding = *lsda++; + uint32_t callSiteTableLength = readULEB128(&lsda); + const uint8_t *callSiteTableStart = lsda; + const uint8_t *callSiteTableEnd = callSiteTableStart + callSiteTableLength; + const uint8_t *p = callSiteTableStart; + while (p < callSiteTableEnd) { + uintptr_t start = readEncodedPointer(&p, callSiteEncoding); + uintptr_t length = readEncodedPointer(&p, callSiteEncoding); + uintptr_t landingPad = readEncodedPointer(&p, callSiteEncoding); + readULEB128(&p); // action value not used for C code + if (landingPad == 0) + continue; // no landing pad for this entry + if ((start <= pcOffset) && (pcOffset < (start + length))) { + // Found landing pad for the PC. + // Set Instruction Pointer to so we re-enter function + // at landing pad. The landing pad is created by the compiler + // to take two parameters in registers. + _Unwind_SetGR(context, __builtin_eh_return_data_regno(0), + (uintptr_t)exceptionObject); + _Unwind_SetGR(context, __builtin_eh_return_data_regno(1), 0); + _Unwind_SetIP(context, (funcStart + landingPad)); + return _URC_INSTALL_CONTEXT; } + } - /* No landing pad found, continue unwinding. */ - return continueUnwind(exceptionObject, context); + // No landing pad found, continue unwinding. + return continueUnwind(exceptionObject, context); } diff --git a/lib/builtins/hexagon/common_entry_exit_abi1.S b/lib/builtins/hexagon/common_entry_exit_abi1.S index d5479d2a52fa..23fed01c6eef 100644 --- a/lib/builtins/hexagon/common_entry_exit_abi1.S +++ b/lib/builtins/hexagon/common_entry_exit_abi1.S @@ -1,14 +1,13 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Functions that implement common sequences in function prologues and epilogues - used to save code size */ +// Functions that implement common sequences in function prologues and epilogues +// used to save code size .macro FUNCTION_BEGIN name .text @@ -33,16 +32,16 @@ -/* Save r25:24 at fp+#-8 and r27:26 at fp+#-16. */ +// Save r25:24 at fp+#-8 and r27:26 at fp+#-16. -/* The compiler knows that the __save_* functions clobber LR. No other - registers should be used without informing the compiler. */ +// The compiler knows that the __save_* functions clobber LR. No other +// registers should be used without informing the compiler. -/* Since we can only issue one store per packet, we don't hurt performance by - simply jumping to the right point in this sequence of stores. */ +// Since we can only issue one store per packet, we don't hurt performance by +// simply jumping to the right point in this sequence of stores. FUNCTION_BEGIN __save_r24_through_r27 memd(fp+#-16) = r27:26 @@ -56,10 +55,10 @@ FUNCTION_END __save_r24_through_r25 -/* For each of the *_before_tailcall functions, jumpr lr is executed in parallel - with deallocframe. That way, the return gets the old value of lr, which is - where these functions need to return, and at the same time, lr gets the value - it needs going into the tail call. */ +// For each of the *_before_tailcall functions, jumpr lr is executed in parallel +// with deallocframe. That way, the return gets the old value of lr, which is +// where these functions need to return, and at the same time, lr gets the value +// it needs going into the tail call. FUNCTION_BEGIN __restore_r24_through_r27_and_deallocframe_before_tailcall r27:26 = memd(fp+#-16) @@ -74,8 +73,8 @@ FUNCTION_END __restore_r24_through_r25_and_deallocframe_before_tailcall -/* Here we use the extra load bandwidth to restore LR early, allowing the return - to occur in parallel with the deallocframe. */ +// Here we use the extra load bandwidth to restore LR early, allowing the return +// to occur in parallel with the deallocframe. FUNCTION_BEGIN __restore_r24_through_r27_and_deallocframe { @@ -92,7 +91,7 @@ FUNCTION_END __restore_r24_through_r27_and_deallocframe -/* Here the load bandwidth is maximized. */ +// Here the load bandwidth is maximized. FUNCTION_BEGIN __restore_r24_through_r25_and_deallocframe { diff --git a/lib/builtins/hexagon/common_entry_exit_abi2.S b/lib/builtins/hexagon/common_entry_exit_abi2.S index 6f470343db49..3b85aea2f600 100644 --- a/lib/builtins/hexagon/common_entry_exit_abi2.S +++ b/lib/builtins/hexagon/common_entry_exit_abi2.S @@ -1,14 +1,13 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Functions that implement common sequences in function prologues and epilogues - used to save code size */ +// Functions that implement common sequences in function prologues and epilogues +// used to save code size .macro FUNCTION_BEGIN name .p2align 2 @@ -33,10 +32,10 @@ -/* Save r17:16 at fp+#-8, r19:18 at fp+#-16, r21:20 at fp+#-24, r23:22 at - fp+#-32, r25:24 at fp+#-40, and r27:26 at fp+#-48. - The compiler knows that the __save_* functions clobber LR. No other - registers should be used without informing the compiler. */ +// Save r17:16 at fp+#-8, r19:18 at fp+#-16, r21:20 at fp+#-24, r23:22 at +// fp+#-32, r25:24 at fp+#-40, and r27:26 at fp+#-48. +// The compiler knows that the __save_* functions clobber LR. No other +// registers should be used without informing the compiler. FUNCTION_BEGIN __save_r16_through_r27 { @@ -107,10 +106,10 @@ FUNCTION_BEGIN __save_r16_through_r17 } FUNCTION_END __save_r16_through_r17 -/* For each of the *_before_tailcall functions, jumpr lr is executed in parallel - with deallocframe. That way, the return gets the old value of lr, which is - where these functions need to return, and at the same time, lr gets the value - it needs going into the tail call. */ +// For each of the *_before_tailcall functions, jumpr lr is executed in parallel +// with deallocframe. That way, the return gets the old value of lr, which is +// where these functions need to return, and at the same time, lr gets the value +// it needs going into the tail call. FUNCTION_BEGIN __restore_r16_through_r27_and_deallocframe_before_tailcall diff --git a/lib/builtins/hexagon/common_entry_exit_legacy.S b/lib/builtins/hexagon/common_entry_exit_legacy.S index 3258f15a3267..8a6044573236 100644 --- a/lib/builtins/hexagon/common_entry_exit_legacy.S +++ b/lib/builtins/hexagon/common_entry_exit_legacy.S @@ -1,15 +1,14 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Functions that implement common sequences in function prologues and epilogues - used to save code size */ +// Functions that implement common sequences in function prologues and epilogues +// used to save code size .macro FUNCTION_BEGIN name .text @@ -34,17 +33,17 @@ -/* Save r27:26 at fp+#-8, r25:24 at fp+#-16, r23:22 at fp+#-24, r21:20 at - fp+#-32, r19:18 at fp+#-40, and r17:16 at fp+#-48. */ +// Save r27:26 at fp+#-8, r25:24 at fp+#-16, r23:22 at fp+#-24, r21:20 at +// fp+#-32, r19:18 at fp+#-40, and r17:16 at fp+#-48. -/* The compiler knows that the __save_* functions clobber LR. No other - registers should be used without informing the compiler. */ +// The compiler knows that the __save_* functions clobber LR. No other +// registers should be used without informing the compiler. -/* Since we can only issue one store per packet, we don't hurt performance by - simply jumping to the right point in this sequence of stores. */ +// Since we can only issue one store per packet, we don't hurt performance by +// simply jumping to the right point in this sequence of stores. FUNCTION_BEGIN __save_r27_through_r16 memd(fp+#-48) = r17:16 @@ -65,10 +64,10 @@ FUNCTION_END __save_r27_through_r24 -/* For each of the *_before_sibcall functions, jumpr lr is executed in parallel - with deallocframe. That way, the return gets the old value of lr, which is - where these functions need to return, and at the same time, lr gets the value - it needs going into the sibcall. */ +// For each of the *_before_sibcall functions, jumpr lr is executed in parallel +// with deallocframe. That way, the return gets the old value of lr, which is +// where these functions need to return, and at the same time, lr gets the value +// it needs going into the sibcall. FUNCTION_BEGIN __restore_r27_through_r20_and_deallocframe_before_sibcall { @@ -108,8 +107,8 @@ FUNCTION_END __restore_r27_through_r26_and_deallocframe_before_sibcall -/* Here we use the extra load bandwidth to restore LR early, allowing the return - to occur in parallel with the deallocframe. */ +// Here we use the extra load bandwidth to restore LR early, allowing the return +// to occur in parallel with the deallocframe. FUNCTION_BEGIN __restore_r27_through_r16_and_deallocframe { @@ -136,7 +135,7 @@ FUNCTION_END __restore_r27_through_r24_and_deallocframe -/* Here the load bandwidth is maximized for all three functions. */ +// Here the load bandwidth is maximized for all three functions. FUNCTION_BEGIN __restore_r27_through_r18_and_deallocframe { diff --git a/lib/builtins/hexagon/dfaddsub.S b/lib/builtins/hexagon/dfaddsub.S index 4173f86a4f54..1b0d34550ff3 100644 --- a/lib/builtins/hexagon/dfaddsub.S +++ b/lib/builtins/hexagon/dfaddsub.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision Multiply */ +// Double Precision Multiply #define A r1:0 #define AH r1 @@ -179,18 +178,17 @@ __hexagon_subdf3: .Ladd_ovf_unf: // Overflow or Denormal is possible // Good news: Underflow flag is not possible! - /* - * ATMP has 2's complement value - * - * EXPA has A's exponent, EXPB has EXPA-BIAS-60 - * - * Convert, extract exponent, add adjustment. - * If > 2046, overflow - * If <= 0, denormal - * - * Note that we've not done our zero check yet, so do that too - * - */ + + // ATMP has 2's complement value + // + // EXPA has A's exponent, EXPB has EXPA-BIAS-60 + // + // Convert, extract exponent, add adjustment. + // If > 2046, overflow + // If <= 0, denormal + // + // Note that we've not done our zero check yet, so do that too + { A = convert_d2df(ATMP) p0 = cmp.eq(ATMPH,#0) diff --git a/lib/builtins/hexagon/dfdiv.S b/lib/builtins/hexagon/dfdiv.S index 0c5dbe272c89..202965ec4789 100644 --- a/lib/builtins/hexagon/dfdiv.S +++ b/lib/builtins/hexagon/dfdiv.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision Divide */ +// Double Precision Divide #define A r1:0 #define AH r1 @@ -237,10 +236,10 @@ __hexagon_divdf3: P_TMP = cmp.gt(EXPA,#0) if (P_TMP.new) jump:nt .Lpossible_unf // round up to normal possible... } - /* Underflow */ - /* We know what the infinite range exponent should be (EXPA) */ - /* Q is 2's complement, PROD is abs(Q) */ - /* Normalize Q, shift right, add a high bit, convert, change exponent */ + // Underflow + // We know what the infinite range exponent should be (EXPA) + // Q is 2's complement, PROD is abs(Q) + // Normalize Q, shift right, add a high bit, convert, change exponent #define FUDGE1 7 // how much to shift right #define FUDGE2 4 // how many guard/round to keep at lsbs @@ -287,8 +286,8 @@ __hexagon_divdf3: .Lpossible_unf: - /* If upper parts of Q were all F's, but abs(A) == 0x00100000_00000000, we rounded up to min_normal */ - /* The answer is correct, but we need to raise Underflow */ + // If upper parts of Q were all F's, but abs(A) == 0x00100000_00000000, we rounded up to min_normal + // The answer is correct, but we need to raise Underflow { B = extractu(A,#63,#0) TMPPAIR = combine(##0x00100000,#0) // min normal @@ -321,9 +320,9 @@ __hexagon_divdf3: } .Ldiv_ovf: - /* - * Raise Overflow, and choose the correct overflow value (saturated normal or infinity) - */ + + // Raise Overflow, and choose the correct overflow value (saturated normal or infinity) + { TMP = USR B = combine(##0x7fefffff,#-1) @@ -389,8 +388,8 @@ __hexagon_divdf3: if (!P_ZERO) jump .Ldiv_zero_result if (!P_INF) jump .Ldiv_inf_result } - /* Now we've narrowed it down to (de)normal / (de)normal */ - /* Set up A/EXPA B/EXPB and go back */ + // Now we've narrowed it down to (de)normal / (de)normal + // Set up A/EXPA B/EXPB and go back #undef P_ZERO #undef P_INF #define P_TMP2 p1 diff --git a/lib/builtins/hexagon/dffma.S b/lib/builtins/hexagon/dffma.S index 97b885a3bf27..c201d3d8be5e 100644 --- a/lib/builtins/hexagon/dffma.S +++ b/lib/builtins/hexagon/dffma.S @@ -1,16 +1,15 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG #define END(TAG) .size TAG,.-TAG -/* Double Precision Multiply */ +// Double Precision Multiply #define A r1:0 @@ -76,33 +75,29 @@ #define SR_ROUND_OFF 22 #endif - /* - * First, classify for normal values, and abort if abnormal - * - * Next, unpack mantissa into 0x1000_0000_0000_0000 + mant<<8 - * - * Since we know that the 2 MSBs of the H registers is zero, we should never carry - * the partial products that involve the H registers - * - * Try to buy X slots, at the expense of latency if needed - * - * We will have PP_HH with the upper bits of the product, PP_LL with the lower - * PP_HH can have a maximum of 0x03FF_FFFF_FFFF_FFFF or thereabouts - * PP_HH can have a minimum of 0x0100_0000_0000_0000 - * - * 0x0100_0000_0000_0000 has EXP of EXPA+EXPB-BIAS - * - * We need to align CTMP. - * If CTMP >> PP, convert PP to 64 bit with sticky, align CTMP, and follow normal add - * If CTMP << PP align CTMP and add 128 bits. Then compute sticky - * If CTMP ~= PP, align CTMP and add 128 bits. May have massive cancellation. - * - * Convert partial product and CTMP to 2's complement prior to addition - * - * After we add, we need to normalize into upper 64 bits, then compute sticky. - * - * - */ + // First, classify for normal values, and abort if abnormal + // + // Next, unpack mantissa into 0x1000_0000_0000_0000 + mant<<8 + // + // Since we know that the 2 MSBs of the H registers is zero, we should never carry + // the partial products that involve the H registers + // + // Try to buy X slots, at the expense of latency if needed + // + // We will have PP_HH with the upper bits of the product, PP_LL with the lower + // PP_HH can have a maximum of 0x03FF_FFFF_FFFF_FFFF or thereabouts + // PP_HH can have a minimum of 0x0100_0000_0000_0000 + // + // 0x0100_0000_0000_0000 has EXP of EXPA+EXPB-BIAS + // + // We need to align CTMP. + // If CTMP >> PP, convert PP to 64 bit with sticky, align CTMP, and follow normal add + // If CTMP << PP align CTMP and add 128 bits. Then compute sticky + // If CTMP ~= PP, align CTMP and add 128 bits. May have massive cancellation. + // + // Convert partial product and CTMP to 2's complement prior to addition + // + // After we add, we need to normalize into upper 64 bits, then compute sticky. .text .global __hexagon_fmadf4 @@ -182,14 +177,12 @@ fma: #define EXPCA r19:18 EXPC = extractu(CH,#EXPBITS,#HI_MANTBITS) } - /* PP_HH:PP_LL now has product */ - /* CTMP is negated */ - /* EXPA,B,C are extracted */ - /* - * We need to negate PP - * Since we will be adding with carry later, if we need to negate, - * just invert all bits now, which we can do conditionally and in parallel - */ + // PP_HH:PP_LL now has product + // CTMP is negated + // EXPA,B,C are extracted + // We need to negate PP + // Since we will be adding with carry later, if we need to negate, + // just invert all bits now, which we can do conditionally and in parallel #define PP_HH_TMP r15:14 #define PP_LL_TMP r7:6 { @@ -274,18 +267,16 @@ fma: PP_HH = add(CTMP,PP_HH,P_CARRY):carry TMP = #62 } - /* - * PP_HH:PP_LL now holds the sum - * We may need to normalize left, up to ??? bits. - * - * I think that if we have massive cancellation, the range we normalize by - * is still limited - */ + // PP_HH:PP_LL now holds the sum + // We may need to normalize left, up to ??? bits. + // + // I think that if we have massive cancellation, the range we normalize by + // is still limited { LEFTSHIFT = add(clb(PP_HH),#-2) if (!cmp.eq(LEFTSHIFT.new,TMP)) jump:t 1f // all sign bits? } - /* We had all sign bits, shift left by 62. */ + // We had all sign bits, shift left by 62. { CTMP = extractu(PP_LL,#62,#2) PP_LL = asl(PP_LL,#62) @@ -330,7 +321,7 @@ fma: if (!P_TMP) dealloc_return // not zero, return } .Ladd_yields_zero: - /* We had full cancellation. Return +/- zero (-0 when round-down) */ + // We had full cancellation. Return +/- zero (-0 when round-down) { TMP = USR A = #0 @@ -408,9 +399,9 @@ fma: EXPA = sub(#1+5,TMP) // Amount to right shift to denormalize p3 = cmp.gt(CTMPH,#-1) } - /* Underflow */ - /* We know that the infinte range exponent should be EXPA */ - /* CTMP is 2's complement, ATMP is abs(CTMP) */ + // Underflow + // We know that the infinte range exponent should be EXPA + // CTMP is 2's complement, ATMP is abs(CTMP) { EXPA = add(EXPA,EXPB) // how much to shift back right ATMP = asl(ATMP,EXPB) // shift left @@ -593,7 +584,7 @@ fma: p1 = dfclass(C,#0x08) if (p1.new) jump:nt .Lfma_inf_plus_inf } - /* A*B is +/- inf, C is finite. Return A */ + // A*B is +/- inf, C is finite. Return A { jumpr r31 } @@ -649,7 +640,7 @@ fma: if (!p0) A = C // If C is not zero, return C if (!p0) jumpr r31 } - /* B has correctly signed zero, C is also zero */ + // B has correctly signed zero, C is also zero .Lzero_plus_zero: { p0 = cmp.eq(B,C) // yes, scalar equals. +0++0 or -0+-0 @@ -674,8 +665,8 @@ fma: #define CTMP r11:10 .falign .Lfma_abnormal_c: - /* We know that AB is normal * normal */ - /* C is not normal: zero, subnormal, inf, or NaN. */ + // We know that AB is normal * normal + // C is not normal: zero, subnormal, inf, or NaN. { p0 = dfclass(C,#0x10) // is C NaN? if (p0.new) jump:nt .Lnan diff --git a/lib/builtins/hexagon/dfminmax.S b/lib/builtins/hexagon/dfminmax.S index 41122911f183..44f031ba11ba 100644 --- a/lib/builtins/hexagon/dfminmax.S +++ b/lib/builtins/hexagon/dfminmax.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,17 +14,14 @@ #define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG #define END(TAG) .size TAG,.-TAG -/* - * Min and Max return A if B is NaN, or B if A is NaN - * Otherwise, they return the smaller or bigger value - * - * If values are equal, we want to favor -0.0 for min and +0.0 for max. - */ +// Min and Max return A if B is NaN, or B if A is NaN +// Otherwise, they return the smaller or bigger value +// +// If values are equal, we want to favor -0.0 for min and +0.0 for max. + +// Compares always return false for NaN +// if (isnan(A)) A = B; if (A > B) A = B will only trigger at most one of those options. -/* - * Compares always return false for NaN - * if (isnan(A)) A = B; if (A > B) A = B will only trigger at most one of those options. - */ .text .global __hexagon_mindf3 .global __hexagon_maxdf3 @@ -51,7 +47,7 @@ fmin: p2 = dfcmp.eq(A,B) // if A == B if (!p2.new) jumpr:t r31 } - /* A == B, return A|B to select -0.0 over 0.0 */ + // A == B, return A|B to select -0.0 over 0.0 { A = or(ATMP,B) jumpr r31 @@ -71,7 +67,7 @@ fmax: p2 = dfcmp.eq(A,B) if (!p2.new) jumpr:t r31 } - /* A == B, return A&B to select 0.0 over -0.0 */ + // A == B, return A&B to select 0.0 over -0.0 { A = and(ATMP,B) jumpr r31 diff --git a/lib/builtins/hexagon/dfmul.S b/lib/builtins/hexagon/dfmul.S index fde6d77bdcff..e6f62c3515f4 100644 --- a/lib/builtins/hexagon/dfmul.S +++ b/lib/builtins/hexagon/dfmul.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision Multiply */ +// Double Precision Multiply #define A r1:0 #define AH r1 #define AL r0 @@ -47,8 +46,8 @@ #define BIAS 1024 #define MANTISSA_TO_INT_BIAS 52 -/* Some constant to adjust normalization amount in error code */ -/* Amount to right shift the partial product to get to a denorm */ +// Some constant to adjust normalization amount in error code +// Amount to right shift the partial product to get to a denorm #define FUDGE 5 #define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG @@ -80,10 +79,10 @@ __hexagon_muldf3: PP_ODD = mpyu(BTMPL,ATMPH) BTMP = insert(ONE,#2,#62) } - /* since we know that the MSB of the H registers is zero, we should never carry */ - /* H <= 2^31-1. L <= 2^32-1. Therefore, HL <= 2^63-2^32-2^31+1 */ - /* Adding 2 HLs, we get 2^64-3*2^32+2 maximum. */ - /* Therefore, we can add 3 2^32-1 values safely without carry. We only need one. */ + // since we know that the MSB of the H registers is zero, we should never carry + // H <= 2^31-1. L <= 2^32-1. Therefore, HL <= 2^63-2^32-2^31+1 + // Adding 2 HLs, we get 2^64-3*2^32+2 maximum. + // Therefore, we can add 3 2^32-1 values safely without carry. We only need one. { PP_LL = mpyu(ATMPL,BTMPL) PP_ODD += mpyu(ATMPL,BTMPH) @@ -99,10 +98,10 @@ __hexagon_muldf3: p1 = cmp.eq(PP_LL_L,#0) // 64 lsb's 0? p1 = cmp.eq(PP_ODD_L,#0) // 64 lsb's 0? } - /* - * PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts - * PP_HH can have a minimum of 0x1000_0000_0000_0000 or so - */ + + // PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts + // PP_HH can have a minimum of 0x1000_0000_0000_0000 or so + #undef PP_ODD #undef PP_ODD_H #undef PP_ODD_L @@ -137,15 +136,15 @@ __hexagon_muldf3: .falign .Lpossible_unf: - /* We end up with a positive exponent */ - /* But we may have rounded up to an exponent of 1. */ - /* If the exponent is 1, if we rounded up to it - * we need to also raise underflow - * Fortunately, this is pretty easy to detect, we must have +/- 0x0010_0000_0000_0000 - * And the PP should also have more than one bit set - */ - /* Note: ATMP should have abs(PP_HH) */ - /* Note: BTMPL should have 0x7FEFFFFF */ + // We end up with a positive exponent + // But we may have rounded up to an exponent of 1. + // If the exponent is 1, if we rounded up to it + // we need to also raise underflow + // Fortunately, this is pretty easy to detect, we must have +/- 0x0010_0000_0000_0000 + // And the PP should also have more than one bit set + // + // Note: ATMP should have abs(PP_HH) + // Note: BTMPL should have 0x7FEFFFFF { p0 = cmp.eq(AL,#0) p0 = bitsclr(AH,BTMPL) @@ -194,29 +193,25 @@ __hexagon_muldf3: BTMPH = sub(EXP0,BTMPH) TMP = #63 // max amount to shift } - /* Underflow */ - /* - * PP_HH has the partial product with sticky LSB. - * PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts - * PP_HH can have a minimum of 0x1000_0000_0000_0000 or so - * The exponent of PP_HH is in EXP1, which is non-positive (0 or negative) - * That's the exponent that happens after the normalization - * - * EXP0 has the exponent that, when added to the normalized value, is out of range. - * - * Strategy: - * - * * Shift down bits, with sticky bit, such that the bits are aligned according - * to the LZ count and appropriate exponent, but not all the way to mantissa - * field, keep around the last few bits. - * * Put a 1 near the MSB - * * Check the LSBs for inexact; if inexact also set underflow - * * Convert [u]d2df -- will correctly round according to rounding mode - * * Replace exponent field with zero - * - * - */ - + // Underflow + // + // PP_HH has the partial product with sticky LSB. + // PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts + // PP_HH can have a minimum of 0x1000_0000_0000_0000 or so + // The exponent of PP_HH is in EXP1, which is non-positive (0 or negative) + // That's the exponent that happens after the normalization + // + // EXP0 has the exponent that, when added to the normalized value, is out of range. + // + // Strategy: + // + // * Shift down bits, with sticky bit, such that the bits are aligned according + // to the LZ count and appropriate exponent, but not all the way to mantissa + // field, keep around the last few bits. + // * Put a 1 near the MSB + // * Check the LSBs for inexact; if inexact also set underflow + // * Convert [u]d2df -- will correctly round according to rounding mode + // * Replace exponent field with zero { BTMPL = #0 // offset for extract diff --git a/lib/builtins/hexagon/dfsqrt.S b/lib/builtins/hexagon/dfsqrt.S index 027d9e1fde43..f1435e868319 100644 --- a/lib/builtins/hexagon/dfsqrt.S +++ b/lib/builtins/hexagon/dfsqrt.S @@ -1,13 +1,12 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -/* Double Precision square root */ +// Double Precision square root #define EXP r28 @@ -169,9 +168,9 @@ __hexagon_sqrt: #define P_CARRY1 p2 #define P_CARRY2 p3 - /* Iteration 0 */ - /* Maybe we can save a cycle by starting with ERROR=asl(fracrad), then as we multiply */ - /* We can shift and subtract instead of shift and add? */ + // Iteration 0 + // Maybe we can save a cycle by starting with ERROR=asl(fracrad), then as we multiply + // We can shift and subtract instead of shift and add? { ERROR = asl(FRACRAD,#15) PROD = mpyu(ROOTHI,ROOTHI) @@ -194,7 +193,7 @@ __hexagon_sqrt: SHIFTAMT = add(SHIFTAMT,#16) ERROR = asl(FRACRAD,#31) // for next iter } - /* Iteration 1 */ + // Iteration 1 { PROD = mpyu(ROOTHI,ROOTHI) ERROR -= mpyu(ROOTHI,ROOTLO) // amount is 31, no shift needed @@ -214,7 +213,7 @@ __hexagon_sqrt: SHIFTAMT = add(SHIFTAMT,#16) ERROR = asl(FRACRAD,#47) // for next iter } - /* Iteration 2 */ + // Iteration 2 { PROD = mpyu(ROOTHI,ROOTHI) } @@ -245,7 +244,7 @@ __hexagon_sqrt: #undef RECIPEST #undef SHIFTAMT #define TWOROOT_LO r9:8 - /* Adjust Root */ + // Adjust Root { HL = mpyu(ROOTHI,ROOTLO) LL = mpyu(ROOTLO,ROOTLO) diff --git a/lib/builtins/hexagon/divdi3.S b/lib/builtins/hexagon/divdi3.S index 49ee8104f305..770601a470f7 100644 --- a/lib/builtins/hexagon/divdi3.S +++ b/lib/builtins/hexagon/divdi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/divsi3.S b/lib/builtins/hexagon/divsi3.S index 8e159baa192f..5f406524e88a 100644 --- a/lib/builtins/hexagon/divsi3.S +++ b/lib/builtins/hexagon/divsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fabs_opt.S b/lib/builtins/hexagon/fabs_opt.S index b09b00734d98..6bf9b84b3d20 100644 --- a/lib/builtins/hexagon/fabs_opt.S +++ b/lib/builtins/hexagon/fabs_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fastmath2_dlib_asm.S b/lib/builtins/hexagon/fastmath2_dlib_asm.S index 9286df06c26d..574a04432f2c 100644 --- a/lib/builtins/hexagon/fastmath2_dlib_asm.S +++ b/lib/builtins/hexagon/fastmath2_dlib_asm.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /* ==================================================================== */ diff --git a/lib/builtins/hexagon/fastmath2_ldlib_asm.S b/lib/builtins/hexagon/fastmath2_ldlib_asm.S index 4192555351a6..cf623f94c849 100644 --- a/lib/builtins/hexagon/fastmath2_ldlib_asm.S +++ b/lib/builtins/hexagon/fastmath2_ldlib_asm.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /* ==================================================================== * diff --git a/lib/builtins/hexagon/fastmath_dlib_asm.S b/lib/builtins/hexagon/fastmath_dlib_asm.S index 215936b783ca..3e59526c1e23 100644 --- a/lib/builtins/hexagon/fastmath_dlib_asm.S +++ b/lib/builtins/hexagon/fastmath_dlib_asm.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /* ==================================================================== */ diff --git a/lib/builtins/hexagon/fma_opt.S b/lib/builtins/hexagon/fma_opt.S index 12378f0da04e..7f566adffd6a 100644 --- a/lib/builtins/hexagon/fma_opt.S +++ b/lib/builtins/hexagon/fma_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fmax_opt.S b/lib/builtins/hexagon/fmax_opt.S index f3a218c9769b..81d711dff8d2 100644 --- a/lib/builtins/hexagon/fmax_opt.S +++ b/lib/builtins/hexagon/fmax_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/fmin_opt.S b/lib/builtins/hexagon/fmin_opt.S index ef9b0ff854a2..d043f1d7a698 100644 --- a/lib/builtins/hexagon/fmin_opt.S +++ b/lib/builtins/hexagon/fmin_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S b/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S index fbe09086cd33..10b81f653343 100644 --- a/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S +++ b/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/builtins/hexagon/memcpy_likely_aligned.S b/lib/builtins/hexagon/memcpy_likely_aligned.S index bbc85c22db08..492298f10326 100644 --- a/lib/builtins/hexagon/memcpy_likely_aligned.S +++ b/lib/builtins/hexagon/memcpy_likely_aligned.S @@ -1,9 +1,8 @@ //===------------------------- memcopy routines ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/moddi3.S b/lib/builtins/hexagon/moddi3.S index 12a0595fe465..d4246b61bdc3 100644 --- a/lib/builtins/hexagon/moddi3.S +++ b/lib/builtins/hexagon/moddi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/modsi3.S b/lib/builtins/hexagon/modsi3.S index 5afda9e2978b..4015d5e06891 100644 --- a/lib/builtins/hexagon/modsi3.S +++ b/lib/builtins/hexagon/modsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/sfdiv_opt.S b/lib/builtins/hexagon/sfdiv_opt.S index 6bdd4808c2b8..7c9ae14b718f 100644 --- a/lib/builtins/hexagon/sfdiv_opt.S +++ b/lib/builtins/hexagon/sfdiv_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/sfsqrt_opt.S b/lib/builtins/hexagon/sfsqrt_opt.S index 7f619002774f..532df9a06d2d 100644 --- a/lib/builtins/hexagon/sfsqrt_opt.S +++ b/lib/builtins/hexagon/sfsqrt_opt.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivdi3.S b/lib/builtins/hexagon/udivdi3.S index 1ca326b75208..23f931d4f587 100644 --- a/lib/builtins/hexagon/udivdi3.S +++ b/lib/builtins/hexagon/udivdi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivmoddi4.S b/lib/builtins/hexagon/udivmoddi4.S index deb5aae0924d..6dbfc59bd650 100644 --- a/lib/builtins/hexagon/udivmoddi4.S +++ b/lib/builtins/hexagon/udivmoddi4.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivmodsi4.S b/lib/builtins/hexagon/udivmodsi4.S index 25bbe7cd5925..9e231212d6c8 100644 --- a/lib/builtins/hexagon/udivmodsi4.S +++ b/lib/builtins/hexagon/udivmodsi4.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/udivsi3.S b/lib/builtins/hexagon/udivsi3.S index 54f0aa409f93..d68599a8e04d 100644 --- a/lib/builtins/hexagon/udivsi3.S +++ b/lib/builtins/hexagon/udivsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/umoddi3.S b/lib/builtins/hexagon/umoddi3.S index f091521414af..646ca128dd96 100644 --- a/lib/builtins/hexagon/umoddi3.S +++ b/lib/builtins/hexagon/umoddi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/hexagon/umodsi3.S b/lib/builtins/hexagon/umodsi3.S index a8270c2030d5..a92394486209 100644 --- a/lib/builtins/hexagon/umodsi3.S +++ b/lib/builtins/hexagon/umodsi3.S @@ -1,9 +1,8 @@ //===----------------------Hexagon builtin routine ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/i386/ashldi3.S b/lib/builtins/i386/ashldi3.S index 6f05dcf74443..7ba912692f0d 100644 --- a/lib/builtins/i386/ashldi3.S +++ b/lib/builtins/i386/ashldi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/ashrdi3.S b/lib/builtins/i386/ashrdi3.S index 206369f360aa..3cca4782aea3 100644 --- a/lib/builtins/i386/ashrdi3.S +++ b/lib/builtins/i386/ashrdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -22,10 +23,10 @@ DEFINE_COMPILERRT_FUNCTION(__ashrdi3) #endif psrlq %xmm2, %xmm0 // unsigned shift input by count - + testl %eax, %eax // check the sign-bit of the input jns 1f // early out for positive inputs - + // If the input is negative, we need to construct the shifted sign bit // to or into the result, as xmm does not have a signed right shift. pcmpeqb %xmm1, %xmm1 // -1ULL @@ -35,7 +36,7 @@ DEFINE_COMPILERRT_FUNCTION(__ashrdi3) psubq %xmm1, %xmm2 // 64 - count psllq %xmm2, %xmm1 // -1 << (64 - count) = leading sign bits por %xmm1, %xmm0 - + // Move the result back to the general purpose registers and return 1: movd %xmm0, %eax psrlq $32, %xmm0 @@ -51,14 +52,14 @@ DEFINE_COMPILERRT_FUNCTION(__ashrdi3) movl 12(%esp), %ecx // Load count movl 8(%esp), %edx // Load high movl 4(%esp), %eax // Load low - + testl $0x20, %ecx // If count >= 32 jnz 1f // goto 1 shrdl %cl, %edx, %eax // right shift low by count sarl %cl, %edx // right shift high by count ret - + 1: movl %edx, %eax // Move high to low sarl $31, %edx // clear high sarl %cl, %eax // shift low by count - 32 diff --git a/lib/builtins/i386/chkstk.S b/lib/builtins/i386/chkstk.S index b59974868f21..f0bea2187457 100644 --- a/lib/builtins/i386/chkstk.S +++ b/lib/builtins/i386/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/chkstk2.S b/lib/builtins/i386/chkstk2.S index 7d65bb088928..5d6cbdfa5c99 100644 --- a/lib/builtins/i386/chkstk2.S +++ b/lib/builtins/i386/chkstk2.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/divdi3.S b/lib/builtins/i386/divdi3.S index 2fb4bdcad90d..09e1e42eb719 100644 --- a/lib/builtins/i386/divdi3.S +++ b/lib/builtins/i386/divdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -22,9 +23,9 @@ .balign 4 DEFINE_COMPILERRT_FUNCTION(__divdi3) -/* This is currently implemented by wrapping the unsigned divide up in an absolute - value, then restoring the correct sign at the end of the computation. This could - certainly be improved upon. */ +// This is currently implemented by wrapping the unsigned divide up in an absolute +// value, then restoring the correct sign at the end of the computation. This could +// certainly be improved upon. pushl %esi movl 20(%esp), %edx // high word of b @@ -38,7 +39,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl %edx, 20(%esp) movl %eax, 16(%esp) // store abs(b) back to stack movl %ecx, %esi // set aside sign of b - + movl 12(%esp), %edx // high word of b movl 8(%esp), %eax // low word of b movl %edx, %ecx @@ -55,11 +56,11 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl 24(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 20(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -68,10 +69,10 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl 16(%esp), %edx // Load the high and low words of a, and jump movl 12(%esp), %eax // to [1] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 1f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 1f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -90,7 +91,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) sbbl $0, %edi // decrement q if remainder is negative xorl %edx, %edx movl %edi, %eax - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -101,8 +102,8 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) retl // Return -1: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +1: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -128,7 +129,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) sbbl $0, %edi // decrement q if remainder is negative xorl %edx, %edx movl %edi, %eax - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -138,8 +139,8 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) popl %esi retl // Return - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 16(%esp), %eax // Find qhi and rhi such that movl 20(%esp), %ecx // @@ -149,7 +150,7 @@ DEFINE_COMPILERRT_FUNCTION(__divdi3) movl 12(%esp), %eax // Find qlo such that divl %ecx // movl %ebx, %edx // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax diff --git a/lib/builtins/i386/floatdidf.S b/lib/builtins/i386/floatdidf.S index d75dfe62d6a7..ab7422c312dc 100644 --- a/lib/builtins/i386/floatdidf.S +++ b/lib/builtins/i386/floatdidf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/floatdisf.S b/lib/builtins/i386/floatdisf.S index 0874eaaa9a98..d91f14e9f330 100644 --- a/lib/builtins/i386/floatdisf.S +++ b/lib/builtins/i386/floatdisf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/floatdixf.S b/lib/builtins/i386/floatdixf.S index 1044ef55a1a8..df70f5f9e6e3 100644 --- a/lib/builtins/i386/floatdixf.S +++ b/lib/builtins/i386/floatdixf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/floatundidf.S b/lib/builtins/i386/floatundidf.S index fe032348e829..8b1b666ce1ec 100644 --- a/lib/builtins/i386/floatundidf.S +++ b/lib/builtins/i386/floatundidf.S @@ -1,9 +1,8 @@ //===-- floatundidf.S - Implement __floatundidf for i386 ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/builtins/i386/floatundisf.S b/lib/builtins/i386/floatundisf.S index 16000b576026..44301719edda 100644 --- a/lib/builtins/i386/floatundisf.S +++ b/lib/builtins/i386/floatundisf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -48,7 +49,7 @@ END_COMPILERRT_FUNCTION(__floatundisf) */ -/* branch-free, x87-free implementation - faster at the expense of code size */ +// branch-free, x87-free implementation - faster at the expense of code size #ifdef __i386__ @@ -78,7 +79,7 @@ DEFINE_COMPILERRT_FUNCTION(__floatundisf) movd 8(%esp), %xmm1 movd 4(%esp), %xmm0 punpckldq %xmm1, %xmm0 - + calll 0f 0: popl %ecx shrl %eax // high 31 bits of input as sint32 diff --git a/lib/builtins/i386/floatundixf.S b/lib/builtins/i386/floatundixf.S index c935670cb52f..30b4d9f4b96c 100644 --- a/lib/builtins/i386/floatundixf.S +++ b/lib/builtins/i386/floatundixf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/i386/lshrdi3.S b/lib/builtins/i386/lshrdi3.S index 53e95cf76527..896633e85d35 100644 --- a/lib/builtins/i386/lshrdi3.S +++ b/lib/builtins/i386/lshrdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -41,14 +42,14 @@ DEFINE_COMPILERRT_FUNCTION(__lshrdi3) movl 12(%esp), %ecx // Load count movl 8(%esp), %edx // Load high movl 4(%esp), %eax // Load low - + testl $0x20, %ecx // If count >= 32 jnz 1f // goto 1 shrdl %cl, %edx, %eax // right shift low by count shrl %cl, %edx // right shift high by count ret - + 1: movl %edx, %eax // Move high to low xorl %edx, %edx // clear high shrl %cl, %eax // shift low by count - 32 diff --git a/lib/builtins/i386/moddi3.S b/lib/builtins/i386/moddi3.S index a5bf9ce8ea0f..4580f20ead15 100644 --- a/lib/builtins/i386/moddi3.S +++ b/lib/builtins/i386/moddi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -23,8 +24,8 @@ .balign 4 DEFINE_COMPILERRT_FUNCTION(__moddi3) -/* This is currently implemented by wrapping the unsigned modulus up in an absolute - value. This could certainly be improved upon. */ +// This is currently implemented by wrapping the unsigned modulus up in an absolute +// value. This could certainly be improved upon. pushl %esi movl 20(%esp), %edx // high word of b @@ -37,7 +38,7 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) sbbl %ecx, %edx // EDX:EAX = abs(b) movl %edx, 20(%esp) movl %eax, 16(%esp) // store abs(b) back to stack - + movl 12(%esp), %edx // high word of b movl 8(%esp), %eax // low word of b movl %edx, %ecx @@ -54,11 +55,11 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) movl 24(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 20(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -67,10 +68,10 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) movl 16(%esp), %edx // Load the high and low words of a, and jump movl 12(%esp), %eax // to [2] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 2f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 2f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -86,13 +87,13 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) movl 28(%esp), %eax imull %edi, %eax // q*bhi subl %eax, %ecx // ECX:EBX = a - q*b - + jnc 1f // if positive, this is the result. addl 24(%esp), %ebx // otherwise adcl 28(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 1: movl %ebx, %eax movl %ecx, %edx - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -102,8 +103,8 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) popl %esi retl // Return -2: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +2: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -132,7 +133,7 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) adcl 28(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 3: movl %ebx, %eax movl %ecx, %edx - + addl %esi, %eax // Restore correct sign to result adcl %esi, %edx xorl %esi, %eax @@ -141,8 +142,8 @@ DEFINE_COMPILERRT_FUNCTION(__moddi3) popl %ebx popl %esi retl // Return - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 16(%esp), %eax // Find qhi and rhi such that movl 20(%esp), %ecx // diff --git a/lib/builtins/i386/muldi3.S b/lib/builtins/i386/muldi3.S index 12394606421c..a898e2414670 100644 --- a/lib/builtins/i386/muldi3.S +++ b/lib/builtins/i386/muldi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -14,15 +15,15 @@ DEFINE_COMPILERRT_FUNCTION(__muldi3) movl 16(%esp), %eax // b.lo movl 12(%esp), %ecx // a.hi imull %eax, %ecx // b.lo * a.hi - + movl 8(%esp), %edx // a.lo movl 20(%esp), %ebx // b.hi imull %edx, %ebx // a.lo * b.hi - + mull %edx // EDX:EAX = a.lo * b.lo addl %ecx, %ebx // EBX = (a.lo*b.hi + a.hi*b.lo) addl %ebx, %edx - + popl %ebx retl END_COMPILERRT_FUNCTION(__muldi3) diff --git a/lib/builtins/i386/udivdi3.S b/lib/builtins/i386/udivdi3.S index 727613639b12..ca390245eabc 100644 --- a/lib/builtins/i386/udivdi3.S +++ b/lib/builtins/i386/udivdi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -26,11 +27,11 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) movl 20(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 16(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -39,10 +40,10 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) movl 12(%esp), %edx // Load the high and low words of a, and jump movl 8(%esp), %eax // to [1] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 1f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 1f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -66,8 +67,8 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) retl -1: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +1: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -97,8 +98,8 @@ DEFINE_COMPILERRT_FUNCTION(__udivdi3) popl %ebx retl - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 12(%esp), %eax // Find qhi and rhi such that movl 16(%esp), %ecx // diff --git a/lib/builtins/i386/umoddi3.S b/lib/builtins/i386/umoddi3.S index 763e821946c0..2717e7e5da68 100644 --- a/lib/builtins/i386/umoddi3.S +++ b/lib/builtins/i386/umoddi3.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -27,11 +28,11 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl 20(%esp), %ebx // Find the index i of the leading bit in b. bsrl %ebx, %ecx // If the high word of b is zero, jump to jz 9f // the code to handle that special case [9]. - - /* High word of b is known to be non-zero on this branch */ - + + // High word of b is known to be non-zero on this branch + movl 16(%esp), %eax // Construct bhi, containing bits [1+i:32+i] of b - + shrl %cl, %eax // Practically, this means that bhi is given by: shrl %eax // notl %ecx // bhi = (high word of b) << (31 - i) | @@ -40,10 +41,10 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl 12(%esp), %edx // Load the high and low words of a, and jump movl 8(%esp), %eax // to [2] if the high word is larger than bhi cmpl %ebx, %edx // to avoid overflowing the upcoming divide. - jae 2f - - /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - + jae 2f + + // High word of a is greater than or equal to (b >> (1 + i)) on this branch + divl %ebx // eax <-- qs, edx <-- r such that ahi:alo = bs*qs + r pushl %edi @@ -59,20 +60,20 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl 24(%esp), %eax imull %edi, %eax // q*bhi subl %eax, %ecx // ECX:EBX = a - q*b - + jnc 1f // if positive, this is the result. addl 20(%esp), %ebx // otherwise adcl 24(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 1: movl %ebx, %eax movl %ecx, %edx - + popl %edi popl %ebx retl -2: /* High word of a is greater than or equal to (b >> (1 + i)) on this branch */ - +2: // High word of a is greater than or equal to (b >> (1 + i)) on this branch + subl %ebx, %edx // subtract bhi from ahi so that divide will not divl %ebx // overflow, and find q and r such that // @@ -101,14 +102,14 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) adcl 24(%esp), %ecx // ECX:EBX = a - (q-1)*b = result 3: movl %ebx, %eax movl %ecx, %edx - + popl %edi popl %ebx retl - -9: /* High word of b is zero on this branch */ + +9: // High word of b is zero on this branch movl 12(%esp), %eax // Find qhi and rhi such that movl 16(%esp), %ecx // @@ -120,7 +121,7 @@ DEFINE_COMPILERRT_FUNCTION(__umoddi3) movl %edx, %eax // rhi:alo = qlo*b + rlo with 0 ≤ rlo < b popl %ebx // xorl %edx, %edx // and return 0:rlo - retl // + retl // END_COMPILERRT_FUNCTION(__umoddi3) #endif // __i386__ diff --git a/lib/builtins/int_endianness.h b/lib/builtins/int_endianness.h index e2586c56bac8..def046c34a62 100644 --- a/lib/builtins/int_endianness.h +++ b/lib/builtins/int_endianness.h @@ -1,51 +1,49 @@ -/* ===-- int_endianness.h - configuration header for compiler-rt ------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file is a configuration header for compiler-rt. - * This file is not part of the interface of this library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_endianness.h - configuration header for compiler-rt -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a configuration header for compiler-rt. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// #ifndef INT_ENDIANNESS_H #define INT_ENDIANNESS_H -#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ defined(__ORDER_LITTLE_ENDIAN__) -/* Clang and GCC provide built-in endianness definitions. */ +// Clang and GCC provide built-in endianness definitions. #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#endif /* __BYTE_ORDER__ */ +#define _YUGA_BIG_ENDIAN 0 +#endif // __BYTE_ORDER__ -#else /* Compilers other than Clang or GCC. */ +#else // Compilers other than Clang or GCC. #if defined(__SVR4) && defined(__sun) #include <sys/byteorder.h> #if defined(_BIG_ENDIAN) #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif defined(_LITTLE_ENDIAN) #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#else /* !_LITTLE_ENDIAN */ +#define _YUGA_BIG_ENDIAN 0 +#else // !_LITTLE_ENDIAN #error "unknown endianness" -#endif /* !_LITTLE_ENDIAN */ +#endif // !_LITTLE_ENDIAN -#endif /* Solaris and AuroraUX. */ +#endif // Solaris and AuroraUX. -/* .. */ +// .. #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \ defined(__minix) @@ -53,64 +51,64 @@ #if _BYTE_ORDER == _BIG_ENDIAN #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif _BYTE_ORDER == _LITTLE_ENDIAN #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#endif /* _BYTE_ORDER */ +#define _YUGA_BIG_ENDIAN 0 +#endif // _BYTE_ORDER -#endif /* *BSD */ +#endif // *BSD #if defined(__OpenBSD__) #include <machine/endian.h> #if _BYTE_ORDER == _BIG_ENDIAN #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #elif _BYTE_ORDER == _LITTLE_ENDIAN #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#endif /* _BYTE_ORDER */ +#define _YUGA_BIG_ENDIAN 0 +#endif // _BYTE_ORDER -#endif /* OpenBSD */ +#endif // OpenBSD -/* .. */ +// .. -/* Mac OSX has __BIG_ENDIAN__ or __LITTLE_ENDIAN__ automatically set by the - * compiler (at least with GCC) */ -#if defined(__APPLE__) || defined(__ellcc__ ) +// Mac OSX has __BIG_ENDIAN__ or __LITTLE_ENDIAN__ automatically set by the +// compiler (at least with GCC) +#if defined(__APPLE__) || defined(__ellcc__) #ifdef __BIG_ENDIAN__ #if __BIG_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 1 #endif -#endif /* __BIG_ENDIAN__ */ +#endif // __BIG_ENDIAN__ #ifdef __LITTLE_ENDIAN__ #if __LITTLE_ENDIAN__ #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 0 #endif -#endif /* __LITTLE_ENDIAN__ */ +#endif // __LITTLE_ENDIAN__ -#endif /* Mac OSX */ +#endif // Mac OSX -/* .. */ +// .. #if defined(_WIN32) #define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 0 -#endif /* Windows */ +#endif // Windows -#endif /* Clang or GCC. */ +#endif // Clang or GCC. -/* . */ +// . #if !defined(_YUGA_LITTLE_ENDIAN) || !defined(_YUGA_BIG_ENDIAN) #error Unable to determine endian -#endif /* Check we found an endianness correctly. */ +#endif // Check we found an endianness correctly. -#endif /* INT_ENDIANNESS_H */ +#endif // INT_ENDIANNESS_H diff --git a/lib/builtins/int_lib.h b/lib/builtins/int_lib.h index fe8a3bdedc0e..3092f68c084a 100644 --- a/lib/builtins/int_lib.h +++ b/lib/builtins/int_lib.h @@ -1,44 +1,33 @@ -/* ===-- int_lib.h - configuration header for compiler-rt -----------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file is a configuration header for compiler-rt. - * This file is not part of the interface of this library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_lib.h - configuration header for compiler-rt -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a configuration header for compiler-rt. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// #ifndef INT_LIB_H #define INT_LIB_H -/* Assumption: Signed integral is 2's complement. */ -/* Assumption: Right shift of signed negative is arithmetic shift. */ -/* Assumption: Endianness is little or big (not mixed). */ +// Assumption: Signed integral is 2's complement. +// Assumption: Right shift of signed negative is arithmetic shift. +// Assumption: Endianness is little or big (not mixed). -#if defined(__ELF__) -#define FNALIAS(alias_name, original_name) \ - void alias_name() __attribute__((__alias__(#original_name))) -#define COMPILER_RT_ALIAS(aliasee) __attribute__((__alias__(#aliasee))) -#else -#define FNALIAS(alias, name) _Pragma("GCC error(\"alias unsupported on this file format\")") -#define COMPILER_RT_ALIAS(aliasee) _Pragma("GCC error(\"alias unsupported on this file format\")") -#endif - -/* ABI macro definitions */ +// ABI macro definitions #if __ARM_EABI__ -# ifdef COMPILER_RT_ARMHF_TARGET -# define COMPILER_RT_ABI -# else -# define COMPILER_RT_ABI __attribute__((__pcs__("aapcs"))) -# endif +#ifdef COMPILER_RT_ARMHF_TARGET +#define COMPILER_RT_ABI +#else +#define COMPILER_RT_ABI __attribute__((__pcs__("aapcs"))) +#endif #else -# define COMPILER_RT_ABI +#define COMPILER_RT_ABI #endif #define AEABI_RTABI __attribute__((__pcs__("aapcs"))) @@ -55,26 +44,44 @@ #define UNUSED __attribute__((unused)) #endif +#define STR(a) #a +#define XSTR(a) STR(a) +#define SYMBOL_NAME(name) XSTR(__USER_LABEL_PREFIX__) #name + +#if defined(__ELF__) || defined(__MINGW32__) || defined(__wasm__) +#define COMPILER_RT_ALIAS(name, aliasname) \ + COMPILER_RT_ABI __typeof(name) aliasname __attribute__((__alias__(#name))); +#elif defined(__APPLE__) +#define COMPILER_RT_ALIAS(name, aliasname) \ + __asm__(".globl " SYMBOL_NAME(aliasname)); \ + __asm__(SYMBOL_NAME(aliasname) " = " SYMBOL_NAME(name)); \ + COMPILER_RT_ABI __typeof(name) aliasname; +#elif defined(_WIN32) +#define COMPILER_RT_ALIAS(name, aliasname) +#else +#error Unsupported target +#endif + #if defined(__NetBSD__) && (defined(_KERNEL) || defined(_STANDALONE)) -/* - * Kernel and boot environment can't use normal headers, - * so use the equivalent system headers. - */ -# include <machine/limits.h> -# include <sys/stdint.h> -# include <sys/types.h> +// +// Kernel and boot environment can't use normal headers, +// so use the equivalent system headers. +// +#include <machine/limits.h> +#include <sys/stdint.h> +#include <sys/types.h> #else -/* Include the standard compiler builtin headers we use functionality from. */ -# include <limits.h> -# include <stdint.h> -# include <stdbool.h> -# include <float.h> +// Include the standard compiler builtin headers we use functionality from. +#include <float.h> +#include <limits.h> +#include <stdbool.h> +#include <stdint.h> #endif -/* Include the commonly used internal type definitions. */ +// Include the commonly used internal type definitions. #include "int_types.h" -/* Include internal utility function declarations. */ +// Include internal utility function declarations. #include "int_util.h" COMPILER_RT_ABI si_int __paritysi2(si_int a); @@ -84,14 +91,14 @@ COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b); COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b); COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d); -COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int* rem); -COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int* rem); +COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int *rem); +COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem); #ifdef CRT_HAS_128BIT COMPILER_RT_ABI si_int __clzti2(ti_int a); -COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int* rem); +COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem); #endif -/* Definitions for builtins unavailable on MSVC */ +// Definitions for builtins unavailable on MSVC #if defined(_MSC_VER) && !defined(__clang__) #include <intrin.h> @@ -129,6 +136,6 @@ uint32_t __inline __builtin_clzll(uint64_t value) { #endif #define __builtin_clzl __builtin_clzll -#endif /* defined(_MSC_VER) && !defined(__clang__) */ +#endif // defined(_MSC_VER) && !defined(__clang__) -#endif /* INT_LIB_H */ +#endif // INT_LIB_H diff --git a/lib/builtins/int_math.h b/lib/builtins/int_math.h index aa3d0721a8ab..58d8990f31b1 100644 --- a/lib/builtins/int_math.h +++ b/lib/builtins/int_math.h @@ -1,34 +1,31 @@ -/* ===-- int_math.h - internal math inlines ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===-----------------------------------------------------------------------=== - * - * This file is not part of the interface of this library. - * - * This file defines substitutes for the libm functions used in some of the - * compiler-rt implementations, defined in such a way that there is not a direct - * dependency on libm or math.h. Instead, we use the compiler builtin versions - * where available. This reduces our dependencies on the system SDK by foisting - * the responsibility onto the compiler. - * - * ===-----------------------------------------------------------------------=== - */ +//===-- int_math.h - internal math inlines --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines substitutes for the libm functions used in some of the +// compiler-rt implementations, defined in such a way that there is not a direct +// dependency on libm or math.h. Instead, we use the compiler builtin versions +// where available. This reduces our dependencies on the system SDK by foisting +// the responsibility onto the compiler. +// +//===----------------------------------------------------------------------===// #ifndef INT_MATH_H #define INT_MATH_H #ifndef __has_builtin -# define __has_builtin(x) 0 +#define __has_builtin(x) 0 #endif #if defined(_MSC_VER) && !defined(__clang__) #include <math.h> #include <stdlib.h> -#include <ymath.h> #endif #if defined(_MSC_VER) && !defined(__clang__) @@ -42,24 +39,23 @@ #define crt_isinf(x) !_finite((x)) #define crt_isnan(x) _isnan((x)) #else -/* Define crt_isfinite in terms of the builtin if available, otherwise provide - * an alternate version in terms of our other functions. This supports some - * versions of GCC which didn't have __builtin_isfinite. - */ +// Define crt_isfinite in terms of the builtin if available, otherwise provide +// an alternate version in terms of our other functions. This supports some +// versions of GCC which didn't have __builtin_isfinite. #if __has_builtin(__builtin_isfinite) -# define crt_isfinite(x) __builtin_isfinite((x)) +#define crt_isfinite(x) __builtin_isfinite((x)) #elif defined(__GNUC__) -# define crt_isfinite(x) \ - __extension__(({ \ - __typeof((x)) x_ = (x); \ - !crt_isinf(x_) && !crt_isnan(x_); \ - })) +#define crt_isfinite(x) \ + __extension__(({ \ + __typeof((x)) x_ = (x); \ + !crt_isinf(x_) && !crt_isnan(x_); \ + })) #else -# error "Do not know how to check for infinity" -#endif /* __has_builtin(__builtin_isfinite) */ +#error "Do not know how to check for infinity" +#endif // __has_builtin(__builtin_isfinite) #define crt_isinf(x) __builtin_isinf((x)) #define crt_isnan(x) __builtin_isnan((x)) -#endif /* _MSC_VER */ +#endif // _MSC_VER #if defined(_MSC_VER) && !defined(__clang__) #define crt_copysign(x, y) copysign((x), (y)) @@ -107,4 +103,4 @@ #define crt_scalbnl(x, y) __builtin_scalbnl((x), (y)) #endif -#endif /* INT_MATH_H */ +#endif // INT_MATH_H diff --git a/lib/builtins/int_types.h b/lib/builtins/int_types.h index 9f8da56cb77b..f89220d54350 100644 --- a/lib/builtins/int_types.h +++ b/lib/builtins/int_types.h @@ -1,63 +1,57 @@ -/* ===-- int_lib.h - configuration header for compiler-rt -----------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file is not part of the interface of this library. - * - * This file defines various standard types, most importantly a number of unions - * used to access parts of larger types. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_lib.h - configuration header for compiler-rt -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines various standard types, most importantly a number of unions +// used to access parts of larger types. +// +//===----------------------------------------------------------------------===// #ifndef INT_TYPES_H #define INT_TYPES_H #include "int_endianness.h" -/* si_int is defined in Linux sysroot's asm-generic/siginfo.h */ +// si_int is defined in Linux sysroot's asm-generic/siginfo.h #ifdef si_int #undef si_int #endif -typedef int si_int; +typedef int si_int; typedef unsigned su_int; -typedef long long di_int; +typedef long long di_int; typedef unsigned long long du_int; -typedef union -{ - di_int all; - struct - { +typedef union { + di_int all; + struct { #if _YUGA_LITTLE_ENDIAN - su_int low; - si_int high; + su_int low; + si_int high; #else - si_int high; - su_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + si_int high; + su_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } dwords; -typedef union -{ - du_int all; - struct - { +typedef union { + du_int all; + struct { #if _YUGA_LITTLE_ENDIAN - su_int low; - su_int high; + su_int low; + su_int high; #else - su_int high; - su_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + su_int high; + su_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } udwords; #if defined(__LP64__) || defined(__wasm__) || defined(__mips64) || \ @@ -65,103 +59,93 @@ typedef union #define CRT_HAS_128BIT #endif -/* MSVC doesn't have a working 128bit integer type. Users should really compile - * compiler-rt with clang, but if they happen to be doing a standalone build for - * asan or something else, disable the 128 bit parts so things sort of work. - */ +// MSVC doesn't have a working 128bit integer type. Users should really compile +// compiler-rt with clang, but if they happen to be doing a standalone build for +// asan or something else, disable the 128 bit parts so things sort of work. #if defined(_MSC_VER) && !defined(__clang__) #undef CRT_HAS_128BIT #endif #ifdef CRT_HAS_128BIT -typedef int ti_int __attribute__ ((mode (TI))); -typedef unsigned tu_int __attribute__ ((mode (TI))); - -typedef union -{ - ti_int all; - struct - { +typedef int ti_int __attribute__((mode(TI))); +typedef unsigned tu_int __attribute__((mode(TI))); + +typedef union { + ti_int all; + struct { #if _YUGA_LITTLE_ENDIAN - du_int low; - di_int high; + du_int low; + di_int high; #else - di_int high; - du_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + di_int high; + du_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } twords; -typedef union -{ - tu_int all; - struct - { +typedef union { + tu_int all; + struct { #if _YUGA_LITTLE_ENDIAN - du_int low; - du_int high; + du_int low; + du_int high; #else - du_int high; - du_int low; -#endif /* _YUGA_LITTLE_ENDIAN */ - }s; + du_int high; + du_int low; +#endif // _YUGA_LITTLE_ENDIAN + } s; } utwords; static __inline ti_int make_ti(di_int h, di_int l) { - twords r; - r.s.high = h; - r.s.low = l; - return r.all; + twords r; + r.s.high = h; + r.s.low = l; + return r.all; } static __inline tu_int make_tu(du_int h, du_int l) { - utwords r; - r.s.high = h; - r.s.low = l; - return r.all; + utwords r; + r.s.high = h; + r.s.low = l; + return r.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT -typedef union -{ - su_int u; - float f; +typedef union { + su_int u; + float f; } float_bits; -typedef union -{ - udwords u; - double f; +typedef union { + udwords u; + double f; } double_bits; -typedef struct -{ +typedef struct { #if _YUGA_LITTLE_ENDIAN - udwords low; - udwords high; + udwords low; + udwords high; #else - udwords high; - udwords low; -#endif /* _YUGA_LITTLE_ENDIAN */ + udwords high; + udwords low; +#endif // _YUGA_LITTLE_ENDIAN } uqwords; -/* Check if the target supports 80 bit extended precision long doubles. - * Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC - * still makes it 80 bits. Clang will match whatever compiler it is trying to - * be compatible with. - */ -#if ((defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)) || \ +// Check if the target supports 80 bit extended precision long doubles. +// Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC +// still makes it 80 bits. Clang will match whatever compiler it is trying to +// be compatible with. +#if ((defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)) || \ defined(__m68k__) || defined(__ia64__) #define HAS_80_BIT_LONG_DOUBLE 1 #else #define HAS_80_BIT_LONG_DOUBLE 0 #endif -typedef union -{ - uqwords u; - long double f; +typedef union { + uqwords u; + long double f; } long_double_bits; #if __STDC_VERSION__ >= 199901L @@ -172,14 +156,19 @@ typedef long double _Complex Lcomplex; #define COMPLEX_REAL(x) __real__(x) #define COMPLEX_IMAGINARY(x) __imag__(x) #else -typedef struct { float real, imaginary; } Fcomplex; +typedef struct { + float real, imaginary; +} Fcomplex; -typedef struct { double real, imaginary; } Dcomplex; +typedef struct { + double real, imaginary; +} Dcomplex; -typedef struct { long double real, imaginary; } Lcomplex; +typedef struct { + long double real, imaginary; +} Lcomplex; #define COMPLEX_REAL(x) (x).real #define COMPLEX_IMAGINARY(x) (x).imaginary #endif -#endif /* INT_TYPES_H */ - +#endif // INT_TYPES_H diff --git a/lib/builtins/int_util.c b/lib/builtins/int_util.c index 752f2015580e..226a6e93440d 100644 --- a/lib/builtins/int_util.c +++ b/lib/builtins/int_util.c @@ -1,25 +1,21 @@ -/* ===-- int_util.c - Implement internal utilities --------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- int_util.c - Implement internal utilities -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -#include "int_util.h" -/* NOTE: The definitions in this file are declared weak because we clients to be - * able to arbitrarily package individual functions into separate .a files. If - * we did not declare these weak, some link situations might end up seeing - * duplicate strong definitions of the same symbol. - * - * We can't use this solution for kernel use (which may not support weak), but - * currently expect that when built for kernel use all the functionality is - * packaged into a single library. - */ +// NOTE: The definitions in this file are declared weak because we clients to be +// able to arbitrarily package individual functions into separate .a files. If +// we did not declare these weak, some link situations might end up seeing +// duplicate strong definitions of the same symbol. +// +// We can't use this solution for kernel use (which may not support weak), but +// currently expect that when built for kernel use all the functionality is +// packaged into a single library. #ifdef KERNEL_USE @@ -33,7 +29,7 @@ void __compilerrt_abort_impl(const char *file, int line, const char *function) { #elif __APPLE__ -/* from libSystem.dylib */ +// from libSystem.dylib NORETURN extern void __assert_rtn(const char *func, const char *file, int line, const char *message); @@ -57,7 +53,7 @@ void __compilerrt_abort_impl(const char *file, int line, const char *function) { #else -/* Get the system definition of abort() */ +// Get the system definition of abort() #include <stdlib.h> #ifndef _WIN32 diff --git a/lib/builtins/int_util.h b/lib/builtins/int_util.h index c3c87381ad8a..5fbdfb57c1e7 100644 --- a/lib/builtins/int_util.h +++ b/lib/builtins/int_util.h @@ -1,25 +1,23 @@ -/* ===-- int_util.h - internal utility functions ----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===-----------------------------------------------------------------------=== - * - * This file is not part of the interface of this library. - * - * This file defines non-inline utilities which are available for use in the - * library. The function definitions themselves are all contained in int_util.c - * which will always be compiled into any compiler-rt library. - * - * ===-----------------------------------------------------------------------=== - */ +//===-- int_util.h - internal utility functions ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is not part of the interface of this library. +// +// This file defines non-inline utilities which are available for use in the +// library. The function definitions themselves are all contained in int_util.c +// which will always be compiled into any compiler-rt library. +// +//===----------------------------------------------------------------------===// #ifndef INT_UTIL_H #define INT_UTIL_H -/** \brief Trigger a program abort (or panic for kernel code). */ +/// \brief Trigger a program abort (or panic for kernel code). #define compilerrt_abort() __compilerrt_abort_impl(__FILE__, __LINE__, __func__) NORETURN void __compilerrt_abort_impl(const char *file, int line, @@ -30,4 +28,4 @@ NORETURN void __compilerrt_abort_impl(const char *file, int line, #define COMPILE_TIME_ASSERT2(expr, cnt) \ typedef char ct_assert_##cnt[(expr) ? 1 : -1] UNUSED -#endif /* INT_UTIL_H */ +#endif // INT_UTIL_H diff --git a/lib/builtins/lshrdi3.c b/lib/builtins/lshrdi3.c index 67b2a7668345..97e08e1e9ba0 100644 --- a/lib/builtins/lshrdi3.c +++ b/lib/builtins/lshrdi3.c @@ -1,45 +1,38 @@ -/* ===-- lshrdi3.c - Implement __lshrdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __lshrdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- lshrdi3.c - Implement __lshrdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __lshrdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: logical a >> b */ +// Returns: logical a >> b -/* Precondition: 0 <= b < bits_in_dword */ +// Precondition: 0 <= b < bits_in_dword -COMPILER_RT_ABI di_int -__lshrdi3(di_int a, si_int b) -{ - const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); - udwords input; - udwords result; - input.all = a; - if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ - { - result.s.high = 0; - result.s.low = input.s.high >> (b - bits_in_word); - } - else /* 0 <= b < bits_in_word */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI di_int __lshrdi3(di_int a, si_int b) { + const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT); + udwords input; + udwords result; + input.all = a; + if (b & bits_in_word) /* bits_in_word <= b < bits_in_dword */ { + result.s.high = 0; + result.s.low = input.s.high >> (b - bits_in_word); + } else /* 0 <= b < bits_in_word */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_word - b)) | (input.s.low >> b); + } + return result.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) COMPILER_RT_ALIAS(__lshrdi3); +COMPILER_RT_ALIAS(__lshrdi3, __aeabi_llsr) #endif diff --git a/lib/builtins/lshrti3.c b/lib/builtins/lshrti3.c index e4170ff84a5a..d00a22095993 100644 --- a/lib/builtins/lshrti3.c +++ b/lib/builtins/lshrti3.c @@ -1,45 +1,38 @@ -/* ===-- lshrti3.c - Implement __lshrti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __lshrti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- lshrti3.c - Implement __lshrti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __lshrti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: logical a >> b */ +// Returns: logical a >> b -/* Precondition: 0 <= b < bits_in_tword */ +// Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int -__lshrti3(ti_int a, si_int b) -{ - const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); - utwords input; - utwords result; - input.all = a; - if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ - { - result.s.high = 0; - result.s.low = input.s.high >> (b - bits_in_dword); - } - else /* 0 <= b < bits_in_dword */ - { - if (b == 0) - return a; - result.s.high = input.s.high >> b; - result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); - } - return result.all; +COMPILER_RT_ABI ti_int __lshrti3(ti_int a, si_int b) { + const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); + utwords input; + utwords result; + input.all = a; + if (b & bits_in_dword) /* bits_in_dword <= b < bits_in_tword */ { + result.s.high = 0; + result.s.low = input.s.high >> (b - bits_in_dword); + } else /* 0 <= b < bits_in_dword */ { + if (b == 0) + return a; + result.s.high = input.s.high >> b; + result.s.low = (input.s.high << (bits_in_dword - b)) | (input.s.low >> b); + } + return result.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mingw_fixfloat.c b/lib/builtins/mingw_fixfloat.c index c462e0dbf654..945be9d4344a 100644 --- a/lib/builtins/mingw_fixfloat.c +++ b/lib/builtins/mingw_fixfloat.c @@ -1,12 +1,10 @@ -/* ===-- mingw_fixfloat.c - Wrap int/float conversions for arm/windows -----=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mingw_fixfloat.c - Wrap int/float conversions for arm/windows -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" diff --git a/lib/builtins/moddi3.c b/lib/builtins/moddi3.c index a04279e3875f..92b0996077c6 100644 --- a/lib/builtins/moddi3.c +++ b/lib/builtins/moddi3.c @@ -1,30 +1,26 @@ -/*===-- moddi3.c - Implement __moddi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __moddi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- moddi3.c - Implement __moddi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __moddi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI di_int -__moddi3(di_int a, di_int b) -{ - const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; - di_int s = b >> bits_in_dword_m1; /* s = b < 0 ? -1 : 0 */ - b = (b ^ s) - s; /* negate if s == -1 */ - s = a >> bits_in_dword_m1; /* s = a < 0 ? -1 : 0 */ - a = (a ^ s) - s; /* negate if s == -1 */ - du_int r; - __udivmoddi4(a, b, &r); - return ((di_int)r ^ s) - s; /* negate if s == -1 */ +COMPILER_RT_ABI di_int __moddi3(di_int a, di_int b) { + const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; + di_int s = b >> bits_in_dword_m1; // s = b < 0 ? -1 : 0 + b = (b ^ s) - s; // negate if s == -1 + s = a >> bits_in_dword_m1; // s = a < 0 ? -1 : 0 + a = (a ^ s) - s; // negate if s == -1 + du_int r; + __udivmoddi4(a, b, &r); + return ((di_int)r ^ s) - s; // negate if s == -1 } diff --git a/lib/builtins/modsi3.c b/lib/builtins/modsi3.c index 86c73ce13777..e443b8a59096 100644 --- a/lib/builtins/modsi3.c +++ b/lib/builtins/modsi3.c @@ -1,23 +1,19 @@ -/* ===-- modsi3.c - Implement __modsi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __modsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- modsi3.c - Implement __modsi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __modsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI si_int -__modsi3(si_int a, si_int b) -{ - return a - __divsi3(a, b) * b; +COMPILER_RT_ABI si_int __modsi3(si_int a, si_int b) { + return a - __divsi3(a, b) * b; } diff --git a/lib/builtins/modti3.c b/lib/builtins/modti3.c index d505c07ac162..d11fe220b769 100644 --- a/lib/builtins/modti3.c +++ b/lib/builtins/modti3.c @@ -1,34 +1,30 @@ -/* ===-- modti3.c - Implement __modti3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __modti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- modti3.c - Implement __modti3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __modti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/*Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI ti_int -__modti3(ti_int a, ti_int b) -{ - const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; - ti_int s = b >> bits_in_tword_m1; /* s = b < 0 ? -1 : 0 */ - b = (b ^ s) - s; /* negate if s == -1 */ - s = a >> bits_in_tword_m1; /* s = a < 0 ? -1 : 0 */ - a = (a ^ s) - s; /* negate if s == -1 */ - tu_int r; - __udivmodti4(a, b, &r); - return ((ti_int)r ^ s) - s; /* negate if s == -1 */ +COMPILER_RT_ABI ti_int __modti3(ti_int a, ti_int b) { + const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1; + ti_int s = b >> bits_in_tword_m1; // s = b < 0 ? -1 : 0 + b = (b ^ s) - s; // negate if s == -1 + s = a >> bits_in_tword_m1; // s = a < 0 ? -1 : 0 + a = (a ^ s) - s; // negate if s == -1 + tu_int r; + __udivmodti4(a, b, &r); + return ((ti_int)r ^ s) - s; // negate if s == -1 } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/muldc3.c b/lib/builtins/muldc3.c index 16d8e98390a3..0ea7041c3706 100644 --- a/lib/builtins/muldc3.c +++ b/lib/builtins/muldc3.c @@ -1,73 +1,65 @@ -/* ===-- muldc3.c - Implement __muldc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __muldc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- muldc3.c - Implement __muldc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __muldc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI Dcomplex -__muldc3(double __a, double __b, double __c, double __d) -{ - double __ac = __a * __c; - double __bd = __b * __d; - double __ad = __a * __d; - double __bc = __b * __c; - Dcomplex z; - COMPLEX_REAL(z) = __ac - __bd; - COMPLEX_IMAGINARY(z) = __ad + __bc; - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - int __recalc = 0; - if (crt_isinf(__a) || crt_isinf(__b)) - { - __a = crt_copysign(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysign(crt_isinf(__b) ? 1 : 0, __b); - if (crt_isnan(__c)) - __c = crt_copysign(0, __c); - if (crt_isnan(__d)) - __d = crt_copysign(0, __d); - __recalc = 1; - } - if (crt_isinf(__c) || crt_isinf(__d)) - { - __c = crt_copysign(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysign(crt_isinf(__d) ? 1 : 0, __d); - if (crt_isnan(__a)) - __a = crt_copysign(0, __a); - if (crt_isnan(__b)) - __b = crt_copysign(0, __b); - __recalc = 1; - } - if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || - crt_isinf(__ad) || crt_isinf(__bc))) - { - if (crt_isnan(__a)) - __a = crt_copysign(0, __a); - if (crt_isnan(__b)) - __b = crt_copysign(0, __b); - if (crt_isnan(__c)) - __c = crt_copysign(0, __c); - if (crt_isnan(__d)) - __d = crt_copysign(0, __d); - __recalc = 1; - } - if (__recalc) - { - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); - } +COMPILER_RT_ABI Dcomplex __muldc3(double __a, double __b, double __c, + double __d) { + double __ac = __a * __c; + double __bd = __b * __d; + double __ad = __a * __d; + double __bc = __b * __c; + Dcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + int __recalc = 0; + if (crt_isinf(__a) || crt_isinf(__b)) { + __a = crt_copysign(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysign(crt_isinf(__b) ? 1 : 0, __b); + if (crt_isnan(__c)) + __c = crt_copysign(0, __c); + if (crt_isnan(__d)) + __d = crt_copysign(0, __d); + __recalc = 1; } - return z; + if (crt_isinf(__c) || crt_isinf(__d)) { + __c = crt_copysign(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysign(crt_isinf(__d) ? 1 : 0, __d); + if (crt_isnan(__a)) + __a = crt_copysign(0, __a); + if (crt_isnan(__b)) + __b = crt_copysign(0, __b); + __recalc = 1; + } + if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || crt_isinf(__ad) || + crt_isinf(__bc))) { + if (crt_isnan(__a)) + __a = crt_copysign(0, __a); + if (crt_isnan(__b)) + __b = crt_copysign(0, __b); + if (crt_isnan(__c)) + __c = crt_copysign(0, __c); + if (crt_isnan(__d)) + __d = crt_copysign(0, __d); + __recalc = 1; + } + if (__recalc) { + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); + } + } + return z; } diff --git a/lib/builtins/muldf3.c b/lib/builtins/muldf3.c index 1bb103e38c13..f64b5228d7b7 100644 --- a/lib/builtins/muldf3.c +++ b/lib/builtins/muldf3.c @@ -1,9 +1,8 @@ //===-- lib/muldf3.c - Double-precision multiplication ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define DOUBLE_PRECISION #include "fp_mul_impl.inc" -COMPILER_RT_ABI fp_t __muldf3(fp_t a, fp_t b) { - return __mulXf3__(a, b); -} +COMPILER_RT_ABI fp_t __muldf3(fp_t a, fp_t b) { return __mulXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) { - return __muldf3(a, b); -} +AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) { return __muldf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__muldf3); +COMPILER_RT_ALIAS(__muldf3, __aeabi_dmul) #endif #endif diff --git a/lib/builtins/muldi3.c b/lib/builtins/muldi3.c index a187315e9165..013f669a1833 100644 --- a/lib/builtins/muldi3.c +++ b/lib/builtins/muldi3.c @@ -1,58 +1,51 @@ -/* ===-- muldi3.c - Implement __muldi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __muldi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- muldi3.c - Implement __muldi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __muldi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -static -di_int -__muldsi3(su_int a, su_int b) -{ - dwords r; - const int bits_in_word_2 = (int)(sizeof(si_int) * CHAR_BIT) / 2; - const su_int lower_mask = (su_int)~0 >> bits_in_word_2; - r.s.low = (a & lower_mask) * (b & lower_mask); - su_int t = r.s.low >> bits_in_word_2; - r.s.low &= lower_mask; - t += (a >> bits_in_word_2) * (b & lower_mask); - r.s.low += (t & lower_mask) << bits_in_word_2; - r.s.high = t >> bits_in_word_2; - t = r.s.low >> bits_in_word_2; - r.s.low &= lower_mask; - t += (b >> bits_in_word_2) * (a & lower_mask); - r.s.low += (t & lower_mask) << bits_in_word_2; - r.s.high += t >> bits_in_word_2; - r.s.high += (a >> bits_in_word_2) * (b >> bits_in_word_2); - return r.all; +static di_int __muldsi3(su_int a, su_int b) { + dwords r; + const int bits_in_word_2 = (int)(sizeof(si_int) * CHAR_BIT) / 2; + const su_int lower_mask = (su_int)~0 >> bits_in_word_2; + r.s.low = (a & lower_mask) * (b & lower_mask); + su_int t = r.s.low >> bits_in_word_2; + r.s.low &= lower_mask; + t += (a >> bits_in_word_2) * (b & lower_mask); + r.s.low += (t & lower_mask) << bits_in_word_2; + r.s.high = t >> bits_in_word_2; + t = r.s.low >> bits_in_word_2; + r.s.low &= lower_mask; + t += (b >> bits_in_word_2) * (a & lower_mask); + r.s.low += (t & lower_mask) << bits_in_word_2; + r.s.high += t >> bits_in_word_2; + r.s.high += (a >> bits_in_word_2) * (b >> bits_in_word_2); + return r.all; } -/* Returns: a * b */ +// Returns: a * b -COMPILER_RT_ABI di_int -__muldi3(di_int a, di_int b) -{ - dwords x; - x.all = a; - dwords y; - y.all = b; - dwords r; - r.all = __muldsi3(x.s.low, y.s.low); - r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; - return r.all; +COMPILER_RT_ABI di_int __muldi3(di_int a, di_int b) { + dwords x; + x.all = a; + dwords y; + y.all = b; + dwords r; + r.all = __muldsi3(x.s.low, y.s.low); + r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; + return r.all; } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) COMPILER_RT_ALIAS(__muldi3); +COMPILER_RT_ALIAS(__muldi3, __aeabi_lmul) #endif diff --git a/lib/builtins/mulodi4.c b/lib/builtins/mulodi4.c index d2fd7db2bcd3..23f5571ac468 100644 --- a/lib/builtins/mulodi4.c +++ b/lib/builtins/mulodi4.c @@ -1,58 +1,49 @@ -/*===-- mulodi4.c - Implement __mulodi4 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulodi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulodi4.c - Implement __mulodi4 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulodi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: sets *overflow to 1 if a * b overflows */ +// Effects: sets *overflow to 1 if a * b overflows -COMPILER_RT_ABI di_int -__mulodi4(di_int a, di_int b, int* overflow) -{ - const int N = (int)(sizeof(di_int) * CHAR_BIT); - const di_int MIN = (di_int)1 << (N-1); - const di_int MAX = ~MIN; - *overflow = 0; - di_int result = a * b; - if (a == MIN) - { - if (b != 0 && b != 1) - *overflow = 1; - return result; - } - if (b == MIN) - { - if (a != 0 && a != 1) - *overflow = 1; - return result; - } - di_int sa = a >> (N - 1); - di_int abs_a = (a ^ sa) - sa; - di_int sb = b >> (N - 1); - di_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return result; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - *overflow = 1; - } - else - { - if (abs_a > MIN / -abs_b) - *overflow = 1; - } +COMPILER_RT_ABI di_int __mulodi4(di_int a, di_int b, int *overflow) { + const int N = (int)(sizeof(di_int) * CHAR_BIT); + const di_int MIN = (di_int)1 << (N - 1); + const di_int MAX = ~MIN; + *overflow = 0; + di_int result = a * b; + if (a == MIN) { + if (b != 0 && b != 1) + *overflow = 1; return result; + } + if (b == MIN) { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + di_int sa = a >> (N - 1); + di_int abs_a = (a ^ sa) - sa; + di_int sb = b >> (N - 1); + di_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) { + if (abs_a > MAX / abs_b) + *overflow = 1; + } else { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; } diff --git a/lib/builtins/mulosi4.c b/lib/builtins/mulosi4.c index 422528085c44..fea4311296f8 100644 --- a/lib/builtins/mulosi4.c +++ b/lib/builtins/mulosi4.c @@ -1,58 +1,49 @@ -/*===-- mulosi4.c - Implement __mulosi4 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulosi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulosi4.c - Implement __mulosi4 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulosi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: sets *overflow to 1 if a * b overflows */ +// Effects: sets *overflow to 1 if a * b overflows -COMPILER_RT_ABI si_int -__mulosi4(si_int a, si_int b, int* overflow) -{ - const int N = (int)(sizeof(si_int) * CHAR_BIT); - const si_int MIN = (si_int)1 << (N-1); - const si_int MAX = ~MIN; - *overflow = 0; - si_int result = a * b; - if (a == MIN) - { - if (b != 0 && b != 1) - *overflow = 1; - return result; - } - if (b == MIN) - { - if (a != 0 && a != 1) - *overflow = 1; - return result; - } - si_int sa = a >> (N - 1); - si_int abs_a = (a ^ sa) - sa; - si_int sb = b >> (N - 1); - si_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return result; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - *overflow = 1; - } - else - { - if (abs_a > MIN / -abs_b) - *overflow = 1; - } +COMPILER_RT_ABI si_int __mulosi4(si_int a, si_int b, int *overflow) { + const int N = (int)(sizeof(si_int) * CHAR_BIT); + const si_int MIN = (si_int)1 << (N - 1); + const si_int MAX = ~MIN; + *overflow = 0; + si_int result = a * b; + if (a == MIN) { + if (b != 0 && b != 1) + *overflow = 1; return result; + } + if (b == MIN) { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + si_int sa = a >> (N - 1); + si_int abs_a = (a ^ sa) - sa; + si_int sb = b >> (N - 1); + si_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) { + if (abs_a > MAX / abs_b) + *overflow = 1; + } else { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; } diff --git a/lib/builtins/muloti4.c b/lib/builtins/muloti4.c index 16b218920321..9bdd5b649908 100644 --- a/lib/builtins/muloti4.c +++ b/lib/builtins/muloti4.c @@ -1,62 +1,53 @@ -/*===-- muloti4.c - Implement __muloti4 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __muloti4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- muloti4.c - Implement __muloti4 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __muloti4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a * b */ +// Returns: a * b -/* Effects: sets *overflow to 1 if a * b overflows */ +// Effects: sets *overflow to 1 if a * b overflows -COMPILER_RT_ABI ti_int -__muloti4(ti_int a, ti_int b, int* overflow) -{ - const int N = (int)(sizeof(ti_int) * CHAR_BIT); - const ti_int MIN = (ti_int)1 << (N-1); - const ti_int MAX = ~MIN; - *overflow = 0; - ti_int result = a * b; - if (a == MIN) - { - if (b != 0 && b != 1) - *overflow = 1; - return result; - } - if (b == MIN) - { - if (a != 0 && a != 1) - *overflow = 1; - return result; - } - ti_int sa = a >> (N - 1); - ti_int abs_a = (a ^ sa) - sa; - ti_int sb = b >> (N - 1); - ti_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return result; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - *overflow = 1; - } - else - { - if (abs_a > MIN / -abs_b) - *overflow = 1; - } +COMPILER_RT_ABI ti_int __muloti4(ti_int a, ti_int b, int *overflow) { + const int N = (int)(sizeof(ti_int) * CHAR_BIT); + const ti_int MIN = (ti_int)1 << (N - 1); + const ti_int MAX = ~MIN; + *overflow = 0; + ti_int result = a * b; + if (a == MIN) { + if (b != 0 && b != 1) + *overflow = 1; return result; + } + if (b == MIN) { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + ti_int sa = a >> (N - 1); + ti_int abs_a = (a ^ sa) - sa; + ti_int sb = b >> (N - 1); + ti_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) { + if (abs_a > MAX / abs_b) + *overflow = 1; + } else { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mulsc3.c b/lib/builtins/mulsc3.c index c89cfd247a15..60653174e440 100644 --- a/lib/builtins/mulsc3.c +++ b/lib/builtins/mulsc3.c @@ -1,73 +1,64 @@ -/* ===-- mulsc3.c - Implement __mulsc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulsc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulsc3.c - Implement __mulsc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulsc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI Fcomplex -__mulsc3(float __a, float __b, float __c, float __d) -{ - float __ac = __a * __c; - float __bd = __b * __d; - float __ad = __a * __d; - float __bc = __b * __c; - Fcomplex z; - COMPLEX_REAL(z) = __ac - __bd; - COMPLEX_IMAGINARY(z) = __ad + __bc; - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - int __recalc = 0; - if (crt_isinf(__a) || crt_isinf(__b)) - { - __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); - if (crt_isnan(__c)) - __c = crt_copysignf(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignf(0, __d); - __recalc = 1; - } - if (crt_isinf(__c) || crt_isinf(__d)) - { - __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); - if (crt_isnan(__a)) - __a = crt_copysignf(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignf(0, __b); - __recalc = 1; - } - if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || - crt_isinf(__ad) || crt_isinf(__bc))) - { - if (crt_isnan(__a)) - __a = crt_copysignf(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignf(0, __b); - if (crt_isnan(__c)) - __c = crt_copysignf(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignf(0, __d); - __recalc = 1; - } - if (__recalc) - { - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); - } +COMPILER_RT_ABI Fcomplex __mulsc3(float __a, float __b, float __c, float __d) { + float __ac = __a * __c; + float __bd = __b * __d; + float __ad = __a * __d; + float __bc = __b * __c; + Fcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + int __recalc = 0; + if (crt_isinf(__a) || crt_isinf(__b)) { + __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); + if (crt_isnan(__c)) + __c = crt_copysignf(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignf(0, __d); + __recalc = 1; } - return z; + if (crt_isinf(__c) || crt_isinf(__d)) { + __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); + if (crt_isnan(__a)) + __a = crt_copysignf(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignf(0, __b); + __recalc = 1; + } + if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || crt_isinf(__ad) || + crt_isinf(__bc))) { + if (crt_isnan(__a)) + __a = crt_copysignf(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignf(0, __b); + if (crt_isnan(__c)) + __c = crt_copysignf(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignf(0, __d); + __recalc = 1; + } + if (__recalc) { + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); + } + } + return z; } diff --git a/lib/builtins/mulsf3.c b/lib/builtins/mulsf3.c index 1e2cf3e717c9..b9cf39abc784 100644 --- a/lib/builtins/mulsf3.c +++ b/lib/builtins/mulsf3.c @@ -1,9 +1,8 @@ //===-- lib/mulsf3.c - Single-precision multiplication ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,16 +14,12 @@ #define SINGLE_PRECISION #include "fp_mul_impl.inc" -COMPILER_RT_ABI fp_t __mulsf3(fp_t a, fp_t b) { - return __mulXf3__(a, b); -} +COMPILER_RT_ABI fp_t __mulsf3(fp_t a, fp_t b) { return __mulXf3__(a, b); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) { - return __mulsf3(a, b); -} +AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) { return __mulsf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__mulsf3); +COMPILER_RT_ALIAS(__mulsf3, __aeabi_fmul) #endif #endif diff --git a/lib/builtins/multc3.c b/lib/builtins/multc3.c index 0518bc2569f1..bb7f6aabfe2c 100644 --- a/lib/builtins/multc3.c +++ b/lib/builtins/multc3.c @@ -1,68 +1,65 @@ -/* ===-- multc3.c - Implement __multc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __multc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- multc3.c - Implement __multc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __multc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI long double _Complex -__multc3(long double a, long double b, long double c, long double d) -{ - long double ac = a * c; - long double bd = b * d; - long double ad = a * d; - long double bc = b * c; - long double _Complex z; - __real__ z = ac - bd; - __imag__ z = ad + bc; - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) { - int recalc = 0; - if (crt_isinf(a) || crt_isinf(b)) { - a = crt_copysignl(crt_isinf(a) ? 1 : 0, a); - b = crt_copysignl(crt_isinf(b) ? 1 : 0, b); - if (crt_isnan(c)) - c = crt_copysignl(0, c); - if (crt_isnan(d)) - d = crt_copysignl(0, d); - recalc = 1; - } - if (crt_isinf(c) || crt_isinf(d)) { - c = crt_copysignl(crt_isinf(c) ? 1 : 0, c); - d = crt_copysignl(crt_isinf(d) ? 1 : 0, d); - if (crt_isnan(a)) - a = crt_copysignl(0, a); - if (crt_isnan(b)) - b = crt_copysignl(0, b); - recalc = 1; - } - if (!recalc && (crt_isinf(ac) || crt_isinf(bd) || - crt_isinf(ad) || crt_isinf(bc))) { - if (crt_isnan(a)) - a = crt_copysignl(0, a); - if (crt_isnan(b)) - b = crt_copysignl(0, b); - if (crt_isnan(c)) - c = crt_copysignl(0, c); - if (crt_isnan(d)) - d = crt_copysignl(0, d); - recalc = 1; - } - if (recalc) { - __real__ z = CRT_INFINITY * (a * c - b * d); - __imag__ z = CRT_INFINITY * (a * d + b * c); - } +COMPILER_RT_ABI long double _Complex __multc3(long double a, long double b, + long double c, long double d) { + long double ac = a * c; + long double bd = b * d; + long double ad = a * d; + long double bc = b * c; + long double _Complex z; + __real__ z = ac - bd; + __imag__ z = ad + bc; + if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) { + int recalc = 0; + if (crt_isinf(a) || crt_isinf(b)) { + a = crt_copysignl(crt_isinf(a) ? 1 : 0, a); + b = crt_copysignl(crt_isinf(b) ? 1 : 0, b); + if (crt_isnan(c)) + c = crt_copysignl(0, c); + if (crt_isnan(d)) + d = crt_copysignl(0, d); + recalc = 1; } - return z; + if (crt_isinf(c) || crt_isinf(d)) { + c = crt_copysignl(crt_isinf(c) ? 1 : 0, c); + d = crt_copysignl(crt_isinf(d) ? 1 : 0, d); + if (crt_isnan(a)) + a = crt_copysignl(0, a); + if (crt_isnan(b)) + b = crt_copysignl(0, b); + recalc = 1; + } + if (!recalc && + (crt_isinf(ac) || crt_isinf(bd) || crt_isinf(ad) || crt_isinf(bc))) { + if (crt_isnan(a)) + a = crt_copysignl(0, a); + if (crt_isnan(b)) + b = crt_copysignl(0, b); + if (crt_isnan(c)) + c = crt_copysignl(0, c); + if (crt_isnan(d)) + d = crt_copysignl(0, d); + recalc = 1; + } + if (recalc) { + __real__ z = CRT_INFINITY * (a * c - b * d); + __imag__ z = CRT_INFINITY * (a * d + b * c); + } + } + return z; } diff --git a/lib/builtins/multf3.c b/lib/builtins/multf3.c index 0b915923ea09..0626fb8c7fc9 100644 --- a/lib/builtins/multf3.c +++ b/lib/builtins/multf3.c @@ -1,9 +1,8 @@ //===-- lib/multf3.c - Quad-precision multiplication --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,6 @@ #if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) #include "fp_mul_impl.inc" -COMPILER_RT_ABI fp_t __multf3(fp_t a, fp_t b) { - return __mulXf3__(a, b); -} +COMPILER_RT_ABI fp_t __multf3(fp_t a, fp_t b) { return __mulXf3__(a, b); } #endif diff --git a/lib/builtins/multi3.c b/lib/builtins/multi3.c index e0d52d430b66..d9d8b59cd3ef 100644 --- a/lib/builtins/multi3.c +++ b/lib/builtins/multi3.c @@ -1,58 +1,51 @@ -/* ===-- multi3.c - Implement __multi3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - - * This file implements __multi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- multi3.c - Implement __multi3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __multi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a * b */ - -static -ti_int -__mulddi3(du_int a, du_int b) -{ - twords r; - const int bits_in_dword_2 = (int)(sizeof(di_int) * CHAR_BIT) / 2; - const du_int lower_mask = (du_int)~0 >> bits_in_dword_2; - r.s.low = (a & lower_mask) * (b & lower_mask); - du_int t = r.s.low >> bits_in_dword_2; - r.s.low &= lower_mask; - t += (a >> bits_in_dword_2) * (b & lower_mask); - r.s.low += (t & lower_mask) << bits_in_dword_2; - r.s.high = t >> bits_in_dword_2; - t = r.s.low >> bits_in_dword_2; - r.s.low &= lower_mask; - t += (b >> bits_in_dword_2) * (a & lower_mask); - r.s.low += (t & lower_mask) << bits_in_dword_2; - r.s.high += t >> bits_in_dword_2; - r.s.high += (a >> bits_in_dword_2) * (b >> bits_in_dword_2); - return r.all; +// Returns: a * b + +static ti_int __mulddi3(du_int a, du_int b) { + twords r; + const int bits_in_dword_2 = (int)(sizeof(di_int) * CHAR_BIT) / 2; + const du_int lower_mask = (du_int)~0 >> bits_in_dword_2; + r.s.low = (a & lower_mask) * (b & lower_mask); + du_int t = r.s.low >> bits_in_dword_2; + r.s.low &= lower_mask; + t += (a >> bits_in_dword_2) * (b & lower_mask); + r.s.low += (t & lower_mask) << bits_in_dword_2; + r.s.high = t >> bits_in_dword_2; + t = r.s.low >> bits_in_dword_2; + r.s.low &= lower_mask; + t += (b >> bits_in_dword_2) * (a & lower_mask); + r.s.low += (t & lower_mask) << bits_in_dword_2; + r.s.high += t >> bits_in_dword_2; + r.s.high += (a >> bits_in_dword_2) * (b >> bits_in_dword_2); + return r.all; } -/* Returns: a * b */ - -COMPILER_RT_ABI ti_int -__multi3(ti_int a, ti_int b) -{ - twords x; - x.all = a; - twords y; - y.all = b; - twords r; - r.all = __mulddi3(x.s.low, y.s.low); - r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; - return r.all; +// Returns: a * b + +COMPILER_RT_ABI ti_int __multi3(ti_int a, ti_int b) { + twords x; + x.all = a; + twords y; + y.all = b; + twords r; + r.all = __mulddi3(x.s.low, y.s.low); + r.s.high += x.s.high * y.s.low + x.s.low * y.s.high; + return r.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mulvdi3.c b/lib/builtins/mulvdi3.c index e63249e0a04c..cecc97ccf22e 100644 --- a/lib/builtins/mulvdi3.c +++ b/lib/builtins/mulvdi3.c @@ -1,56 +1,47 @@ -/*===-- mulvdi3.c - Implement __mulvdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulvdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulvdi3.c - Implement __mulvdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulvdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: aborts if a * b overflows */ +// Effects: aborts if a * b overflows -COMPILER_RT_ABI di_int -__mulvdi3(di_int a, di_int b) -{ - const int N = (int)(sizeof(di_int) * CHAR_BIT); - const di_int MIN = (di_int)1 << (N-1); - const di_int MAX = ~MIN; - if (a == MIN) - { - if (b == 0 || b == 1) - return a * b; - compilerrt_abort(); - } - if (b == MIN) - { - if (a == 0 || a == 1) - return a * b; - compilerrt_abort(); - } - di_int sa = a >> (N - 1); - di_int abs_a = (a ^ sa) - sa; - di_int sb = b >> (N - 1); - di_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return a * b; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - compilerrt_abort(); - } - else - { - if (abs_a > MIN / -abs_b) - compilerrt_abort(); - } +COMPILER_RT_ABI di_int __mulvdi3(di_int a, di_int b) { + const int N = (int)(sizeof(di_int) * CHAR_BIT); + const di_int MIN = (di_int)1 << (N - 1); + const di_int MAX = ~MIN; + if (a == MIN) { + if (b == 0 || b == 1) + return a * b; + compilerrt_abort(); + } + if (b == MIN) { + if (a == 0 || a == 1) + return a * b; + compilerrt_abort(); + } + di_int sa = a >> (N - 1); + di_int abs_a = (a ^ sa) - sa; + di_int sb = b >> (N - 1); + di_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) return a * b; + if (sa == sb) { + if (abs_a > MAX / abs_b) + compilerrt_abort(); + } else { + if (abs_a > MIN / -abs_b) + compilerrt_abort(); + } + return a * b; } diff --git a/lib/builtins/mulvsi3.c b/lib/builtins/mulvsi3.c index 74ea4f2da226..0d6b18ad01a4 100644 --- a/lib/builtins/mulvsi3.c +++ b/lib/builtins/mulvsi3.c @@ -1,56 +1,47 @@ -/* ===-- mulvsi3.c - Implement __mulvsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulvsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulvsi3.c - Implement __mulvsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulvsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a * b */ +// Returns: a * b -/* Effects: aborts if a * b overflows */ +// Effects: aborts if a * b overflows -COMPILER_RT_ABI si_int -__mulvsi3(si_int a, si_int b) -{ - const int N = (int)(sizeof(si_int) * CHAR_BIT); - const si_int MIN = (si_int)1 << (N-1); - const si_int MAX = ~MIN; - if (a == MIN) - { - if (b == 0 || b == 1) - return a * b; - compilerrt_abort(); - } - if (b == MIN) - { - if (a == 0 || a == 1) - return a * b; - compilerrt_abort(); - } - si_int sa = a >> (N - 1); - si_int abs_a = (a ^ sa) - sa; - si_int sb = b >> (N - 1); - si_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return a * b; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - compilerrt_abort(); - } - else - { - if (abs_a > MIN / -abs_b) - compilerrt_abort(); - } +COMPILER_RT_ABI si_int __mulvsi3(si_int a, si_int b) { + const int N = (int)(sizeof(si_int) * CHAR_BIT); + const si_int MIN = (si_int)1 << (N - 1); + const si_int MAX = ~MIN; + if (a == MIN) { + if (b == 0 || b == 1) + return a * b; + compilerrt_abort(); + } + if (b == MIN) { + if (a == 0 || a == 1) + return a * b; + compilerrt_abort(); + } + si_int sa = a >> (N - 1); + si_int abs_a = (a ^ sa) - sa; + si_int sb = b >> (N - 1); + si_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) return a * b; + if (sa == sb) { + if (abs_a > MAX / abs_b) + compilerrt_abort(); + } else { + if (abs_a > MIN / -abs_b) + compilerrt_abort(); + } + return a * b; } diff --git a/lib/builtins/mulvti3.c b/lib/builtins/mulvti3.c index f4c7d1612ba9..03963a0ca694 100644 --- a/lib/builtins/mulvti3.c +++ b/lib/builtins/mulvti3.c @@ -1,60 +1,51 @@ -/* ===-- mulvti3.c - Implement __mulvti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulvti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulvti3.c - Implement __mulvti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulvti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a * b */ +// Returns: a * b -/* Effects: aborts if a * b overflows */ +// Effects: aborts if a * b overflows -COMPILER_RT_ABI ti_int -__mulvti3(ti_int a, ti_int b) -{ - const int N = (int)(sizeof(ti_int) * CHAR_BIT); - const ti_int MIN = (ti_int)1 << (N-1); - const ti_int MAX = ~MIN; - if (a == MIN) - { - if (b == 0 || b == 1) - return a * b; - compilerrt_abort(); - } - if (b == MIN) - { - if (a == 0 || a == 1) - return a * b; - compilerrt_abort(); - } - ti_int sa = a >> (N - 1); - ti_int abs_a = (a ^ sa) - sa; - ti_int sb = b >> (N - 1); - ti_int abs_b = (b ^ sb) - sb; - if (abs_a < 2 || abs_b < 2) - return a * b; - if (sa == sb) - { - if (abs_a > MAX / abs_b) - compilerrt_abort(); - } - else - { - if (abs_a > MIN / -abs_b) - compilerrt_abort(); - } +COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) { + const int N = (int)(sizeof(ti_int) * CHAR_BIT); + const ti_int MIN = (ti_int)1 << (N - 1); + const ti_int MAX = ~MIN; + if (a == MIN) { + if (b == 0 || b == 1) + return a * b; + compilerrt_abort(); + } + if (b == MIN) { + if (a == 0 || a == 1) + return a * b; + compilerrt_abort(); + } + ti_int sa = a >> (N - 1); + ti_int abs_a = (a ^ sa) - sa; + ti_int sb = b >> (N - 1); + ti_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) return a * b; + if (sa == sb) { + if (abs_a > MAX / abs_b) + compilerrt_abort(); + } else { + if (abs_a > MIN / -abs_b) + compilerrt_abort(); + } + return a * b; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/mulxc3.c b/lib/builtins/mulxc3.c index ba3221691821..2f7f14c28453 100644 --- a/lib/builtins/mulxc3.c +++ b/lib/builtins/mulxc3.c @@ -1,77 +1,69 @@ -/* ===-- mulxc3.c - Implement __mulxc3 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __mulxc3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- mulxc3.c - Implement __mulxc3 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __mulxc3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" #include "int_math.h" -/* Returns: the product of a + ib and c + id */ +// Returns: the product of a + ib and c + id -COMPILER_RT_ABI Lcomplex -__mulxc3(long double __a, long double __b, long double __c, long double __d) -{ - long double __ac = __a * __c; - long double __bd = __b * __d; - long double __ad = __a * __d; - long double __bc = __b * __c; - Lcomplex z; - COMPLEX_REAL(z) = __ac - __bd; - COMPLEX_IMAGINARY(z) = __ad + __bc; - if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) - { - int __recalc = 0; - if (crt_isinf(__a) || crt_isinf(__b)) - { - __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); - __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); - if (crt_isnan(__c)) - __c = crt_copysignl(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignl(0, __d); - __recalc = 1; - } - if (crt_isinf(__c) || crt_isinf(__d)) - { - __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); - __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); - if (crt_isnan(__a)) - __a = crt_copysignl(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignl(0, __b); - __recalc = 1; - } - if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || - crt_isinf(__ad) || crt_isinf(__bc))) - { - if (crt_isnan(__a)) - __a = crt_copysignl(0, __a); - if (crt_isnan(__b)) - __b = crt_copysignl(0, __b); - if (crt_isnan(__c)) - __c = crt_copysignl(0, __c); - if (crt_isnan(__d)) - __d = crt_copysignl(0, __d); - __recalc = 1; - } - if (__recalc) - { - COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); - COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); - } +COMPILER_RT_ABI Lcomplex __mulxc3(long double __a, long double __b, + long double __c, long double __d) { + long double __ac = __a * __c; + long double __bd = __b * __d; + long double __ad = __a * __d; + long double __bc = __b * __c; + Lcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { + int __recalc = 0; + if (crt_isinf(__a) || crt_isinf(__b)) { + __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); + __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); + if (crt_isnan(__c)) + __c = crt_copysignl(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignl(0, __d); + __recalc = 1; } - return z; + if (crt_isinf(__c) || crt_isinf(__d)) { + __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); + __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); + if (crt_isnan(__a)) + __a = crt_copysignl(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignl(0, __b); + __recalc = 1; + } + if (!__recalc && (crt_isinf(__ac) || crt_isinf(__bd) || crt_isinf(__ad) || + crt_isinf(__bc))) { + if (crt_isnan(__a)) + __a = crt_copysignl(0, __a); + if (crt_isnan(__b)) + __b = crt_copysignl(0, __b); + if (crt_isnan(__c)) + __c = crt_copysignl(0, __c); + if (crt_isnan(__d)) + __d = crt_copysignl(0, __d); + __recalc = 1; + } + if (__recalc) { + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); + } + } + return z; } #endif diff --git a/lib/builtins/negdf2.c b/lib/builtins/negdf2.c index f0bfaad24743..f9ceaa374542 100644 --- a/lib/builtins/negdf2.c +++ b/lib/builtins/negdf2.c @@ -1,9 +1,8 @@ //===-- lib/negdf2.c - double-precision negation ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,17 +13,12 @@ #define DOUBLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__negdf2(fp_t a) { - return fromRep(toRep(a) ^ signBit); -} +COMPILER_RT_ABI fp_t __negdf2(fp_t a) { return fromRep(toRep(a) ^ signBit); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_dneg(fp_t a) { - return __negdf2(a); -} +AEABI_RTABI fp_t __aeabi_dneg(fp_t a) { return __negdf2(a); } #else -AEABI_RTABI fp_t __aeabi_dneg(fp_t a) COMPILER_RT_ALIAS(__negdf2); +COMPILER_RT_ALIAS(__negdf2, __aeabi_dneg) #endif #endif diff --git a/lib/builtins/negdi2.c b/lib/builtins/negdi2.c index 3d49ba2899d2..5a525d4b0e55 100644 --- a/lib/builtins/negdi2.c +++ b/lib/builtins/negdi2.c @@ -1,26 +1,21 @@ -/* ===-- negdi2.c - Implement __negdi2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negdi2.c - Implement __negdi2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: -a */ +// Returns: -a -COMPILER_RT_ABI di_int -__negdi2(di_int a) -{ - /* Note: this routine is here for API compatibility; any sane compiler - * should expand it inline. - */ - return -a; +COMPILER_RT_ABI di_int __negdi2(di_int a) { + // Note: this routine is here for API compatibility; any sane compiler + // should expand it inline. + return -a; } diff --git a/lib/builtins/negsf2.c b/lib/builtins/negsf2.c index 05c97d4d5a11..d59dfe7cf91b 100644 --- a/lib/builtins/negsf2.c +++ b/lib/builtins/negsf2.c @@ -1,9 +1,8 @@ //===-- lib/negsf2.c - single-precision negation ------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,17 +13,12 @@ #define SINGLE_PRECISION #include "fp_lib.h" -COMPILER_RT_ABI fp_t -__negsf2(fp_t a) { - return fromRep(toRep(a) ^ signBit); -} +COMPILER_RT_ABI fp_t __negsf2(fp_t a) { return fromRep(toRep(a) ^ signBit); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fneg(fp_t a) { - return __negsf2(a); -} +AEABI_RTABI fp_t __aeabi_fneg(fp_t a) { return __negsf2(a); } #else -AEABI_RTABI fp_t __aeabi_fneg(fp_t a) COMPILER_RT_ALIAS(__negsf2); +COMPILER_RT_ALIAS(__negsf2, __aeabi_fneg) #endif #endif diff --git a/lib/builtins/negti2.c b/lib/builtins/negti2.c index 9b00b303f856..d52ba4e13a46 100644 --- a/lib/builtins/negti2.c +++ b/lib/builtins/negti2.c @@ -1,30 +1,25 @@ -/* ===-- negti2.c - Implement __negti2 -------------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negti2.c - Implement __negti2 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: -a */ +// Returns: -a -COMPILER_RT_ABI ti_int -__negti2(ti_int a) -{ - /* Note: this routine is here for API compatibility; any sane compiler - * should expand it inline. - */ - return -a; +COMPILER_RT_ABI ti_int __negti2(ti_int a) { + // Note: this routine is here for API compatibility; any sane compiler + // should expand it inline. + return -a; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/negvdi2.c b/lib/builtins/negvdi2.c index e336ecf28f0d..5c52b3ec2aa6 100644 --- a/lib/builtins/negvdi2.c +++ b/lib/builtins/negvdi2.c @@ -1,28 +1,24 @@ -/* ===-- negvdi2.c - Implement __negvdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negvdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negvdi2.c - Implement __negvdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negvdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: -a */ +// Returns: -a -/* Effects: aborts if -a overflows */ +// Effects: aborts if -a overflows -COMPILER_RT_ABI di_int -__negvdi2(di_int a) -{ - const di_int MIN = (di_int)1 << ((int)(sizeof(di_int) * CHAR_BIT)-1); - if (a == MIN) - compilerrt_abort(); - return -a; +COMPILER_RT_ABI di_int __negvdi2(di_int a) { + const di_int MIN = (di_int)1 << ((int)(sizeof(di_int) * CHAR_BIT) - 1); + if (a == MIN) + compilerrt_abort(); + return -a; } diff --git a/lib/builtins/negvsi2.c b/lib/builtins/negvsi2.c index b9e93fef06c5..cccdee6dc5e5 100644 --- a/lib/builtins/negvsi2.c +++ b/lib/builtins/negvsi2.c @@ -1,28 +1,24 @@ -/* ===-- negvsi2.c - Implement __negvsi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __negvsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- negvsi2.c - Implement __negvsi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negvsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: -a */ +// Returns: -a -/* Effects: aborts if -a overflows */ +// Effects: aborts if -a overflows -COMPILER_RT_ABI si_int -__negvsi2(si_int a) -{ - const si_int MIN = (si_int)1 << ((int)(sizeof(si_int) * CHAR_BIT)-1); - if (a == MIN) - compilerrt_abort(); - return -a; +COMPILER_RT_ABI si_int __negvsi2(si_int a) { + const si_int MIN = (si_int)1 << ((int)(sizeof(si_int) * CHAR_BIT) - 1); + if (a == MIN) + compilerrt_abort(); + return -a; } diff --git a/lib/builtins/negvti2.c b/lib/builtins/negvti2.c index 85f9f7d19d9d..8f92e1046d0c 100644 --- a/lib/builtins/negvti2.c +++ b/lib/builtins/negvti2.c @@ -1,32 +1,28 @@ -/*===-- negvti2.c - Implement __negvti2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - *===----------------------------------------------------------------------=== - * - *This file implements __negvti2 for the compiler_rt library. - * - *===----------------------------------------------------------------------=== - */ +//===-- negvti2.c - Implement __negvti2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __negvti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: -a */ +// Returns: -a -/* Effects: aborts if -a overflows */ +// Effects: aborts if -a overflows -COMPILER_RT_ABI ti_int -__negvti2(ti_int a) -{ - const ti_int MIN = (ti_int)1 << ((int)(sizeof(ti_int) * CHAR_BIT)-1); - if (a == MIN) - compilerrt_abort(); - return -a; +COMPILER_RT_ABI ti_int __negvti2(ti_int a) { + const ti_int MIN = (ti_int)1 << ((int)(sizeof(ti_int) * CHAR_BIT) - 1); + if (a == MIN) + compilerrt_abort(); + return -a; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/os_version_check.c b/lib/builtins/os_version_check.c index e0d40edc7e38..3794b979434c 100644 --- a/lib/builtins/os_version_check.c +++ b/lib/builtins/os_version_check.c @@ -1,17 +1,15 @@ -/* ===-- os_version_check.c - OS version checking -------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements the function __isOSVersionAtLeast, used by - * Objective-C's @available - * - * ===----------------------------------------------------------------------=== - */ +//===-- os_version_check.c - OS version checking -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the function __isOSVersionAtLeast, used by +// Objective-C's @available +// +//===----------------------------------------------------------------------===// #ifdef __APPLE__ @@ -23,15 +21,15 @@ #include <stdlib.h> #include <string.h> -/* These three variables hold the host's OS version. */ +// These three variables hold the host's OS version. static int32_t GlobalMajor, GlobalMinor, GlobalSubminor; static dispatch_once_t DispatchOnceCounter; -/* We can't include <CoreFoundation/CoreFoundation.h> directly from here, so - * just forward declare everything that we need from it. */ +// We can't include <CoreFoundation/CoreFoundation.h> directly from here, so +// just forward declare everything that we need from it. typedef const void *CFDataRef, *CFAllocatorRef, *CFPropertyListRef, - *CFStringRef, *CFDictionaryRef, *CFTypeRef, *CFErrorRef; + *CFStringRef, *CFDictionaryRef, *CFTypeRef, *CFErrorRef; #if __LLP64__ typedef unsigned long long CFTypeID; @@ -48,9 +46,9 @@ typedef _Bool Boolean; typedef CFIndex CFPropertyListFormat; typedef uint32_t CFStringEncoding; -/* kCFStringEncodingASCII analog. */ +// kCFStringEncodingASCII analog. #define CF_STRING_ENCODING_ASCII 0x0600 -/* kCFStringEncodingUTF8 analog. */ +// kCFStringEncodingUTF8 analog. #define CF_STRING_ENCODING_UTF8 0x08000100 #define CF_PROPERTY_LIST_IMMUTABLE 0 @@ -74,10 +72,10 @@ typedef Boolean (*CFStringGetCStringFuncTy)(CFStringRef, char *, CFIndex, CFStringEncoding); typedef void (*CFReleaseFuncTy)(CFTypeRef); -/* Find and parse the SystemVersion.plist file. */ +// Find and parse the SystemVersion.plist file. static void parseSystemVersionPList(void *Unused) { (void)Unused; - /* Load CoreFoundation dynamically */ + // Load CoreFoundation dynamically const void *NullAllocator = dlsym(RTLD_DEFAULT, "kCFAllocatorNull"); if (!NullAllocator) return; @@ -88,18 +86,18 @@ static void parseSystemVersionPList(void *Unused) { if (!CFDataCreateWithBytesNoCopyFunc) return; CFPropertyListCreateWithDataFuncTy CFPropertyListCreateWithDataFunc = - (CFPropertyListCreateWithDataFuncTy)dlsym( - RTLD_DEFAULT, "CFPropertyListCreateWithData"); -/* CFPropertyListCreateWithData was introduced only in macOS 10.6+, so it - * will be NULL on earlier OS versions. */ + (CFPropertyListCreateWithDataFuncTy)dlsym(RTLD_DEFAULT, + "CFPropertyListCreateWithData"); +// CFPropertyListCreateWithData was introduced only in macOS 10.6+, so it +// will be NULL on earlier OS versions. #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" CFPropertyListCreateFromXMLDataFuncTy CFPropertyListCreateFromXMLDataFunc = (CFPropertyListCreateFromXMLDataFuncTy)dlsym( RTLD_DEFAULT, "CFPropertyListCreateFromXMLData"); #pragma clang diagnostic pop - /* CFPropertyListCreateFromXMLDataFunc is deprecated in macOS 10.10, so it - * might be NULL in future OS versions. */ + // CFPropertyListCreateFromXMLDataFunc is deprecated in macOS 10.10, so it + // might be NULL in future OS versions. if (!CFPropertyListCreateWithDataFunc && !CFPropertyListCreateFromXMLDataFunc) return; CFStringCreateWithCStringNoCopyFuncTy CFStringCreateWithCStringNoCopyFunc = @@ -143,7 +141,7 @@ static void parseSystemVersionPList(void *Unused) { if (!PropertyList) return; - /* Dynamically allocated stuff. */ + // Dynamically allocated stuff. CFDictionaryRef PListRef = NULL; CFDataRef FileContentsRef = NULL; UInt8 *PListBuf = NULL; @@ -162,8 +160,8 @@ static void parseSystemVersionPList(void *Unused) { if (NumRead != (size_t)PListFileSize) goto Fail; - /* Get the file buffer into CF's format. We pass in a null allocator here * - * because we free PListBuf ourselves */ + // Get the file buffer into CF's format. We pass in a null allocator here * + // because we free PListBuf ourselves FileContentsRef = (*CFDataCreateWithBytesNoCopyFunc)( NULL, PListBuf, (CFIndex)NumRead, AllocatorNull); if (!FileContentsRef) @@ -204,7 +202,7 @@ Fail: } int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) { - /* Populate the global version variables, if they haven't already. */ + // Populate the global version variables, if they haven't already. dispatch_once_f(&DispatchOnceCounter, NULL, parseSystemVersionPList); if (Major < GlobalMajor) @@ -220,7 +218,7 @@ int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) { #else -/* Silence an empty translation unit warning. */ +// Silence an empty translation unit warning. typedef int unused; #endif diff --git a/lib/builtins/paritydi2.c b/lib/builtins/paritydi2.c index 8ea5ab4214e9..dd9d45e63ea4 100644 --- a/lib/builtins/paritydi2.c +++ b/lib/builtins/paritydi2.c @@ -1,25 +1,21 @@ -/* ===-- paritydi2.c - Implement __paritydi2 -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __paritydi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- paritydi2.c - Implement __paritydi2 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __paritydi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: 1 if number of bits is odd else returns 0 */ +// Returns: 1 if number of bits is odd else returns 0 -COMPILER_RT_ABI si_int -__paritydi2(di_int a) -{ - dwords x; - x.all = a; - return __paritysi2(x.s.high ^ x.s.low); +COMPILER_RT_ABI si_int __paritydi2(di_int a) { + dwords x; + x.all = a; + return __paritysi2(x.s.high ^ x.s.low); } diff --git a/lib/builtins/paritysi2.c b/lib/builtins/paritysi2.c index 599984663849..3efa961f2f85 100644 --- a/lib/builtins/paritysi2.c +++ b/lib/builtins/paritysi2.c @@ -1,27 +1,23 @@ -/* ===-- paritysi2.c - Implement __paritysi2 -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __paritysi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- paritysi2.c - Implement __paritysi2 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __paritysi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: 1 if number of bits is odd else returns 0 */ +// Returns: 1 if number of bits is odd else returns 0 -COMPILER_RT_ABI si_int -__paritysi2(si_int a) -{ - su_int x = (su_int)a; - x ^= x >> 16; - x ^= x >> 8; - x ^= x >> 4; - return (0x6996 >> (x & 0xF)) & 1; +COMPILER_RT_ABI si_int __paritysi2(si_int a) { + su_int x = (su_int)a; + x ^= x >> 16; + x ^= x >> 8; + x ^= x >> 4; + return (0x6996 >> (x & 0xF)) & 1; } diff --git a/lib/builtins/parityti2.c b/lib/builtins/parityti2.c index 5a4fe492486f..f3942ba8378c 100644 --- a/lib/builtins/parityti2.c +++ b/lib/builtins/parityti2.c @@ -1,29 +1,25 @@ -/* ===-- parityti2.c - Implement __parityti2 -------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __parityti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- parityti2.c - Implement __parityti2 -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __parityti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: 1 if number of bits is odd else returns 0 */ +// Returns: 1 if number of bits is odd else returns 0 -COMPILER_RT_ABI si_int -__parityti2(ti_int a) -{ - twords x; - x.all = a; - return __paritydi2(x.s.high ^ x.s.low); +COMPILER_RT_ABI si_int __parityti2(ti_int a) { + twords x; + x.all = a; + return __paritydi2(x.s.high ^ x.s.low); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/popcountdi2.c b/lib/builtins/popcountdi2.c index 5e8a62f075eb..9bbc39c6608a 100644 --- a/lib/builtins/popcountdi2.c +++ b/lib/builtins/popcountdi2.c @@ -1,36 +1,32 @@ -/* ===-- popcountdi2.c - Implement __popcountdi2 ----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __popcountdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- popcountdi2.c - Implement __popcountdi2 ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: count of 1 bits */ +// Returns: count of 1 bits -COMPILER_RT_ABI si_int -__popcountdi2(di_int a) -{ - du_int x2 = (du_int)a; - x2 = x2 - ((x2 >> 1) & 0x5555555555555555uLL); - /* Every 2 bits holds the sum of every pair of bits (32) */ - x2 = ((x2 >> 2) & 0x3333333333333333uLL) + (x2 & 0x3333333333333333uLL); - /* Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (16) */ - x2 = (x2 + (x2 >> 4)) & 0x0F0F0F0F0F0F0F0FuLL; - /* Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (8) */ - su_int x = (su_int)(x2 + (x2 >> 32)); - /* The lower 32 bits hold four 16 bit sums (5 significant bits). */ - /* Upper 32 bits are garbage */ - x = x + (x >> 16); - /* The lower 16 bits hold two 32 bit sums (6 significant bits). */ - /* Upper 16 bits are garbage */ - return (x + (x >> 8)) & 0x0000007F; /* (7 significant bits) */ +COMPILER_RT_ABI si_int __popcountdi2(di_int a) { + du_int x2 = (du_int)a; + x2 = x2 - ((x2 >> 1) & 0x5555555555555555uLL); + // Every 2 bits holds the sum of every pair of bits (32) + x2 = ((x2 >> 2) & 0x3333333333333333uLL) + (x2 & 0x3333333333333333uLL); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (16) + x2 = (x2 + (x2 >> 4)) & 0x0F0F0F0F0F0F0F0FuLL; + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (8) + su_int x = (su_int)(x2 + (x2 >> 32)); + // The lower 32 bits hold four 16 bit sums (5 significant bits). + // Upper 32 bits are garbage + x = x + (x >> 16); + // The lower 16 bits hold two 32 bit sums (6 significant bits). + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0x0000007F; // (7 significant bits) } diff --git a/lib/builtins/popcountsi2.c b/lib/builtins/popcountsi2.c index 44544ff49890..75e592a778d9 100644 --- a/lib/builtins/popcountsi2.c +++ b/lib/builtins/popcountsi2.c @@ -1,33 +1,29 @@ -/* ===-- popcountsi2.c - Implement __popcountsi2 ---------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __popcountsi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- popcountsi2.c - Implement __popcountsi2 ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountsi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: count of 1 bits */ +// Returns: count of 1 bits -COMPILER_RT_ABI si_int -__popcountsi2(si_int a) -{ - su_int x = (su_int)a; - x = x - ((x >> 1) & 0x55555555); - /* Every 2 bits holds the sum of every pair of bits */ - x = ((x >> 2) & 0x33333333) + (x & 0x33333333); - /* Every 4 bits holds the sum of every 4-set of bits (3 significant bits) */ - x = (x + (x >> 4)) & 0x0F0F0F0F; - /* Every 8 bits holds the sum of every 8-set of bits (4 significant bits) */ - x = (x + (x >> 16)); - /* The lower 16 bits hold two 8 bit sums (5 significant bits).*/ - /* Upper 16 bits are garbage */ - return (x + (x >> 8)) & 0x0000003F; /* (6 significant bits) */ +COMPILER_RT_ABI si_int __popcountsi2(si_int a) { + su_int x = (su_int)a; + x = x - ((x >> 1) & 0x55555555); + // Every 2 bits holds the sum of every pair of bits + x = ((x >> 2) & 0x33333333) + (x & 0x33333333); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) + x = (x + (x >> 4)) & 0x0F0F0F0F; + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) + x = (x + (x >> 16)); + // The lower 16 bits hold two 8 bit sums (5 significant bits). + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0x0000003F; // (6 significant bits) } diff --git a/lib/builtins/popcountti2.c b/lib/builtins/popcountti2.c index 7451bbb286b7..853fd722309e 100644 --- a/lib/builtins/popcountti2.c +++ b/lib/builtins/popcountti2.c @@ -1,44 +1,43 @@ -/* ===-- popcountti2.c - Implement __popcountti2 ----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __popcountti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- popcountti2.c - Implement __popcountti2 +//----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __popcountti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: count of 1 bits */ +// Returns: count of 1 bits -COMPILER_RT_ABI si_int -__popcountti2(ti_int a) -{ - tu_int x3 = (tu_int)a; - x3 = x3 - ((x3 >> 1) & (((tu_int)0x5555555555555555uLL << 64) | - 0x5555555555555555uLL)); - /* Every 2 bits holds the sum of every pair of bits (64) */ - x3 = ((x3 >> 2) & (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)) - + (x3 & (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)); - /* Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (32) */ - x3 = (x3 + (x3 >> 4)) - & (((tu_int)0x0F0F0F0F0F0F0F0FuLL << 64) | 0x0F0F0F0F0F0F0F0FuLL); - /* Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (16) */ - du_int x2 = (du_int)(x3 + (x3 >> 64)); - /* Every 8 bits holds the sum of every 8-set of bits (5 significant bits) (8) */ - su_int x = (su_int)(x2 + (x2 >> 32)); - /* Every 8 bits holds the sum of every 8-set of bits (6 significant bits) (4) */ - x = x + (x >> 16); - /* Every 8 bits holds the sum of every 8-set of bits (7 significant bits) (2) */ - /* Upper 16 bits are garbage */ - return (x + (x >> 8)) & 0xFF; /* (8 significant bits) */ +COMPILER_RT_ABI si_int __popcountti2(ti_int a) { + tu_int x3 = (tu_int)a; + x3 = x3 - ((x3 >> 1) & + (((tu_int)0x5555555555555555uLL << 64) | 0x5555555555555555uLL)); + // Every 2 bits holds the sum of every pair of bits (64) + x3 = ((x3 >> 2) & + (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)) + + (x3 & (((tu_int)0x3333333333333333uLL << 64) | 0x3333333333333333uLL)); + // Every 4 bits holds the sum of every 4-set of bits (3 significant bits) (32) + x3 = (x3 + (x3 >> 4)) & + (((tu_int)0x0F0F0F0F0F0F0F0FuLL << 64) | 0x0F0F0F0F0F0F0F0FuLL); + // Every 8 bits holds the sum of every 8-set of bits (4 significant bits) (16) + du_int x2 = (du_int)(x3 + (x3 >> 64)); + // Every 8 bits holds the sum of every 8-set of bits (5 significant bits) (8) + su_int x = (su_int)(x2 + (x2 >> 32)); + // Every 8 bits holds the sum of every 8-set of bits (6 significant bits) (4) + x = x + (x >> 16); + // Every 8 bits holds the sum of every 8-set of bits (7 significant bits) (2) + // + // Upper 16 bits are garbage + return (x + (x >> 8)) & 0xFF; // (8 significant bits) } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/powidf2.c b/lib/builtins/powidf2.c index ac13b172b043..9697588484e7 100644 --- a/lib/builtins/powidf2.c +++ b/lib/builtins/powidf2.c @@ -1,34 +1,29 @@ -/* ===-- powidf2.cpp - Implement __powidf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powidf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powidf2.cpp - Implement __powidf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powidf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI double -__powidf2(double a, si_int b) -{ - const int recip = b < 0; - double r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI double __powidf2(double a, si_int b) { + const int recip = b < 0; + double r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } diff --git a/lib/builtins/powisf2.c b/lib/builtins/powisf2.c index 0c400ec6dd6a..469402348825 100644 --- a/lib/builtins/powisf2.c +++ b/lib/builtins/powisf2.c @@ -1,34 +1,29 @@ -/*===-- powisf2.cpp - Implement __powisf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powisf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powisf2.cpp - Implement __powisf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powisf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI float -__powisf2(float a, si_int b) -{ - const int recip = b < 0; - float r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI float __powisf2(float a, si_int b) { + const int recip = b < 0; + float r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } diff --git a/lib/builtins/powitf2.c b/lib/builtins/powitf2.c index 172f29f58f25..fcbdb4c2ee2a 100644 --- a/lib/builtins/powitf2.c +++ b/lib/builtins/powitf2.c @@ -1,38 +1,33 @@ -/* ===-- powitf2.cpp - Implement __powitf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powitf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powitf2.cpp - Implement __powitf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powitf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #if _ARCH_PPC -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI long double -__powitf2(long double a, si_int b) -{ - const int recip = b < 0; - long double r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI long double __powitf2(long double a, si_int b) { + const int recip = b < 0; + long double r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } #endif diff --git a/lib/builtins/powixf2.c b/lib/builtins/powixf2.c index 0fd96e503e7b..b7b52095afa1 100644 --- a/lib/builtins/powixf2.c +++ b/lib/builtins/powixf2.c @@ -1,38 +1,33 @@ -/* ===-- powixf2.cpp - Implement __powixf2 ---------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __powixf2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- powixf2.cpp - Implement __powixf2 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __powixf2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #if !_ARCH_PPC #include "int_lib.h" -/* Returns: a ^ b */ +// Returns: a ^ b -COMPILER_RT_ABI long double -__powixf2(long double a, si_int b) -{ - const int recip = b < 0; - long double r = 1; - while (1) - { - if (b & 1) - r *= a; - b /= 2; - if (b == 0) - break; - a *= a; - } - return recip ? 1/r : r; +COMPILER_RT_ABI long double __powixf2(long double a, si_int b) { + const int recip = b < 0; + long double r = 1; + while (1) { + if (b & 1) + r *= a; + b /= 2; + if (b == 0) + break; + a *= a; + } + return recip ? 1 / r : r; } #endif diff --git a/lib/builtins/ppc/DD.h b/lib/builtins/ppc/DD.h index 3e5f9e58c138..8f31a962fc77 100644 --- a/lib/builtins/ppc/DD.h +++ b/lib/builtins/ppc/DD.h @@ -4,20 +4,20 @@ #include "../int_lib.h" typedef union { - long double ld; - struct { - double hi; - double lo; - }s; + long double ld; + struct { + double hi; + double lo; + } s; } DD; -typedef union { - double d; - uint64_t x; +typedef union { + double d; + uint64_t x; } doublebits; -#define LOWORDER(xy,xHi,xLo,yHi,yLo) \ - (((((xHi)*(yHi) - (xy)) + (xHi)*(yLo)) + (xLo)*(yHi)) + (xLo)*(yLo)) +#define LOWORDER(xy, xHi, xLo, yHi, yLo) \ + (((((xHi) * (yHi) - (xy)) + (xHi) * (yLo)) + (xLo) * (yHi)) + (xLo) * (yLo)) static __inline ALWAYS_INLINE double local_fabs(double x) { doublebits result = {.d = x}; @@ -42,4 +42,4 @@ long double __gcc_qsub(long double, long double); long double __gcc_qmul(long double, long double); long double __gcc_qdiv(long double, long double); -#endif /* COMPILERRT_DD_HEADER */ +#endif // COMPILERRT_DD_HEADER diff --git a/lib/builtins/ppc/divtc3.c b/lib/builtins/ppc/divtc3.c index ef532b841145..afaccf5a8fd6 100644 --- a/lib/builtins/ppc/divtc3.c +++ b/lib/builtins/ppc/divtc3.c @@ -1,9 +1,9 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "DD.h" #include "../int_math.h" +#include "DD.h" // Use DOUBLE_PRECISION because the soft-fp method we use is logb (on the upper // half of the long doubles), even though this file defines complex division for // 128-bit floats. @@ -12,86 +12,85 @@ #if !defined(CRT_INFINITY) && defined(HUGE_VAL) #define CRT_INFINITY HUGE_VAL -#endif /* CRT_INFINITY */ +#endif // CRT_INFINITY + +#define makeFinite(x) \ + { \ + (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ + (x).s.lo = 0.0; \ + } + +long double _Complex __divtc3(long double a, long double b, long double c, + long double d) { + DD cDD = {.ld = c}; + DD dDD = {.ld = d}; + + int ilogbw = 0; + const double logbw = + __compiler_rt_logb(crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi))); + + if (crt_isfinite(logbw)) { + ilogbw = (int)logbw; + + cDD.s.hi = crt_scalbn(cDD.s.hi, -ilogbw); + cDD.s.lo = crt_scalbn(cDD.s.lo, -ilogbw); + dDD.s.hi = crt_scalbn(dDD.s.hi, -ilogbw); + dDD.s.lo = crt_scalbn(dDD.s.lo, -ilogbw); + } + + const long double denom = + __gcc_qadd(__gcc_qmul(cDD.ld, cDD.ld), __gcc_qmul(dDD.ld, dDD.ld)); + const long double realNumerator = + __gcc_qadd(__gcc_qmul(a, cDD.ld), __gcc_qmul(b, dDD.ld)); + const long double imagNumerator = + __gcc_qsub(__gcc_qmul(b, cDD.ld), __gcc_qmul(a, dDD.ld)); + + DD real = {.ld = __gcc_qdiv(realNumerator, denom)}; + DD imag = {.ld = __gcc_qdiv(imagNumerator, denom)}; + + real.s.hi = crt_scalbn(real.s.hi, -ilogbw); + real.s.lo = crt_scalbn(real.s.lo, -ilogbw); + imag.s.hi = crt_scalbn(imag.s.hi, -ilogbw); + imag.s.lo = crt_scalbn(imag.s.lo, -ilogbw); -#define makeFinite(x) { \ - (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ - (x).s.lo = 0.0; \ + if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) { + DD aDD = {.ld = a}; + DD bDD = {.ld = b}; + DD rDD = {.ld = denom}; + + if ((rDD.s.hi == 0.0) && (!crt_isnan(aDD.s.hi) || !crt_isnan(bDD.s.hi))) { + real.s.hi = crt_copysign(CRT_INFINITY, cDD.s.hi) * aDD.s.hi; + real.s.lo = 0.0; + imag.s.hi = crt_copysign(CRT_INFINITY, cDD.s.hi) * bDD.s.hi; + imag.s.lo = 0.0; + } + + else if ((crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) && + crt_isfinite(cDD.s.hi) && crt_isfinite(dDD.s.hi)) { + makeFinite(aDD); + makeFinite(bDD); + real.s.hi = CRT_INFINITY * (aDD.s.hi * cDD.s.hi + bDD.s.hi * dDD.s.hi); + real.s.lo = 0.0; + imag.s.hi = CRT_INFINITY * (bDD.s.hi * cDD.s.hi - aDD.s.hi * dDD.s.hi); + imag.s.lo = 0.0; + } + + else if ((crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) && + crt_isfinite(aDD.s.hi) && crt_isfinite(bDD.s.hi)) { + makeFinite(cDD); + makeFinite(dDD); + real.s.hi = + crt_copysign(0.0, (aDD.s.hi * cDD.s.hi + bDD.s.hi * dDD.s.hi)); + real.s.lo = 0.0; + imag.s.hi = + crt_copysign(0.0, (bDD.s.hi * cDD.s.hi - aDD.s.hi * dDD.s.hi)); + imag.s.lo = 0.0; + } } -long double _Complex -__divtc3(long double a, long double b, long double c, long double d) -{ - DD cDD = { .ld = c }; - DD dDD = { .ld = d }; - - int ilogbw = 0; - const double logbw = __compiler_rt_logb( - crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi))); - - if (crt_isfinite(logbw)) - { - ilogbw = (int)logbw; - - cDD.s.hi = crt_scalbn(cDD.s.hi, -ilogbw); - cDD.s.lo = crt_scalbn(cDD.s.lo, -ilogbw); - dDD.s.hi = crt_scalbn(dDD.s.hi, -ilogbw); - dDD.s.lo = crt_scalbn(dDD.s.lo, -ilogbw); - } - - const long double denom = __gcc_qadd(__gcc_qmul(cDD.ld, cDD.ld), __gcc_qmul(dDD.ld, dDD.ld)); - const long double realNumerator = __gcc_qadd(__gcc_qmul(a,cDD.ld), __gcc_qmul(b,dDD.ld)); - const long double imagNumerator = __gcc_qsub(__gcc_qmul(b,cDD.ld), __gcc_qmul(a,dDD.ld)); - - DD real = { .ld = __gcc_qdiv(realNumerator, denom) }; - DD imag = { .ld = __gcc_qdiv(imagNumerator, denom) }; - - real.s.hi = crt_scalbn(real.s.hi, -ilogbw); - real.s.lo = crt_scalbn(real.s.lo, -ilogbw); - imag.s.hi = crt_scalbn(imag.s.hi, -ilogbw); - imag.s.lo = crt_scalbn(imag.s.lo, -ilogbw); - - if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) - { - DD aDD = { .ld = a }; - DD bDD = { .ld = b }; - DD rDD = { .ld = denom }; - - if ((rDD.s.hi == 0.0) && (!crt_isnan(aDD.s.hi) || - !crt_isnan(bDD.s.hi))) - { - real.s.hi = crt_copysign(CRT_INFINITY,cDD.s.hi) * aDD.s.hi; - real.s.lo = 0.0; - imag.s.hi = crt_copysign(CRT_INFINITY,cDD.s.hi) * bDD.s.hi; - imag.s.lo = 0.0; - } - - else if ((crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) && - crt_isfinite(cDD.s.hi) && crt_isfinite(dDD.s.hi)) - { - makeFinite(aDD); - makeFinite(bDD); - real.s.hi = CRT_INFINITY * (aDD.s.hi*cDD.s.hi + bDD.s.hi*dDD.s.hi); - real.s.lo = 0.0; - imag.s.hi = CRT_INFINITY * (bDD.s.hi*cDD.s.hi - aDD.s.hi*dDD.s.hi); - imag.s.lo = 0.0; - } - - else if ((crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) && - crt_isfinite(aDD.s.hi) && crt_isfinite(bDD.s.hi)) - { - makeFinite(cDD); - makeFinite(dDD); - real.s.hi = crt_copysign(0.0,(aDD.s.hi*cDD.s.hi + bDD.s.hi*dDD.s.hi)); - real.s.lo = 0.0; - imag.s.hi = crt_copysign(0.0,(bDD.s.hi*cDD.s.hi - aDD.s.hi*dDD.s.hi)); - imag.s.lo = 0.0; - } - } - - long double _Complex z; - __real__ z = real.ld; - __imag__ z = imag.ld; - - return z; + long double _Complex z; + __real__ z = real.ld; + __imag__ z = imag.ld; + + return z; } diff --git a/lib/builtins/ppc/fixtfdi.c b/lib/builtins/ppc/fixtfdi.c index 2c7c0f8e279f..a97aaf095846 100644 --- a/lib/builtins/ppc/fixtfdi.c +++ b/lib/builtins/ppc/fixtfdi.c @@ -1,104 +1,98 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* int64_t __fixunstfdi(long double x); - * This file implements the PowerPC 128-bit double-double -> int64_t conversion - */ +// int64_t __fixunstfdi(long double x); +// This file implements the PowerPC 128-bit double-double -> int64_t conversion -#include "DD.h" #include "../int_math.h" +#include "DD.h" + +uint64_t __fixtfdi(long double input) { + const DD x = {.ld = input}; + const doublebits hibits = {.d = x.s.hi}; + + const uint32_t absHighWord = + (uint32_t)(hibits.x >> 32) & UINT32_C(0x7fffffff); + const uint32_t absHighWordMinusOne = absHighWord - UINT32_C(0x3ff00000); + + // If (1.0 - tiny) <= input < 0x1.0p63: + if (UINT32_C(0x03f00000) > absHighWordMinusOne) { + // Do an unsigned conversion of the absolute value, then restore the sign. + const int unbiasedHeadExponent = absHighWordMinusOne >> 20; + + int64_t result = hibits.x & INT64_C(0x000fffffffffffff); // mantissa(hi) + result |= INT64_C(0x0010000000000000); // matissa(hi) with implicit bit + result <<= 10; // mantissa(hi) with one zero preceding bit. + + const int64_t hiNegationMask = ((int64_t)(hibits.x)) >> 63; + + // If the tail is non-zero, we need to patch in the tail bits. + if (0.0 != x.s.lo) { + const doublebits lobits = {.d = x.s.lo}; + int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); + tailMantissa |= INT64_C(0x0010000000000000); + + // At this point we have the mantissa of |tail| + // We need to negate it if head and tail have different signs. + const int64_t loNegationMask = ((int64_t)(lobits.x)) >> 63; + const int64_t negationMask = loNegationMask ^ hiNegationMask; + tailMantissa = (tailMantissa ^ negationMask) - negationMask; + + // Now we have the mantissa of tail as a signed 2s-complement integer + + const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; + + // Shift the tail mantissa into the right position, accounting for the + // bias of 10 that we shifted the head mantissa by. + tailMantissa >>= + (unbiasedHeadExponent - (biasedTailExponent - (1023 - 10))); + + result += tailMantissa; + } + + result >>= (62 - unbiasedHeadExponent); + + // Restore the sign of the result and return + result = (result ^ hiNegationMask) - hiNegationMask; + return result; + } + + // Edge cases handled here: + + // |x| < 1, result is zero. + if (1.0 > crt_fabs(x.s.hi)) + return INT64_C(0); + + // x very close to INT64_MIN, care must be taken to see which side we are on. + if (x.s.hi == -0x1.0p63) { + + int64_t result = INT64_MIN; + + if (0.0 < x.s.lo) { + // If the tail is positive, the correct result is something other than + // INT64_MIN. we'll need to figure out what it is. + + const doublebits lobits = {.d = x.s.lo}; + int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); + tailMantissa |= INT64_C(0x0010000000000000); + + // Now we negate the tailMantissa + tailMantissa = (tailMantissa ^ INT64_C(-1)) + INT64_C(1); + + // And shift it by the appropriate amount + const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; + tailMantissa >>= 1075 - biasedTailExponent; + + result -= tailMantissa; + } + + return result; + } -uint64_t __fixtfdi(long double input) -{ - const DD x = { .ld = input }; - const doublebits hibits = { .d = x.s.hi }; - - const uint32_t absHighWord = (uint32_t)(hibits.x >> 32) & UINT32_C(0x7fffffff); - const uint32_t absHighWordMinusOne = absHighWord - UINT32_C(0x3ff00000); - - /* If (1.0 - tiny) <= input < 0x1.0p63: */ - if (UINT32_C(0x03f00000) > absHighWordMinusOne) - { - /* Do an unsigned conversion of the absolute value, then restore the sign. */ - const int unbiasedHeadExponent = absHighWordMinusOne >> 20; - - int64_t result = hibits.x & INT64_C(0x000fffffffffffff); /* mantissa(hi) */ - result |= INT64_C(0x0010000000000000); /* matissa(hi) with implicit bit */ - result <<= 10; /* mantissa(hi) with one zero preceding bit. */ - - const int64_t hiNegationMask = ((int64_t)(hibits.x)) >> 63; - - /* If the tail is non-zero, we need to patch in the tail bits. */ - if (0.0 != x.s.lo) - { - const doublebits lobits = { .d = x.s.lo }; - int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); - tailMantissa |= INT64_C(0x0010000000000000); - - /* At this point we have the mantissa of |tail| */ - /* We need to negate it if head and tail have different signs. */ - const int64_t loNegationMask = ((int64_t)(lobits.x)) >> 63; - const int64_t negationMask = loNegationMask ^ hiNegationMask; - tailMantissa = (tailMantissa ^ negationMask) - negationMask; - - /* Now we have the mantissa of tail as a signed 2s-complement integer */ - - const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; - - /* Shift the tail mantissa into the right position, accounting for the - * bias of 10 that we shifted the head mantissa by. - */ - tailMantissa >>= (unbiasedHeadExponent - (biasedTailExponent - (1023 - 10))); - - result += tailMantissa; - } - - result >>= (62 - unbiasedHeadExponent); - - /* Restore the sign of the result and return */ - result = (result ^ hiNegationMask) - hiNegationMask; - return result; - - } - - /* Edge cases handled here: */ - - /* |x| < 1, result is zero. */ - if (1.0 > crt_fabs(x.s.hi)) - return INT64_C(0); - - /* x very close to INT64_MIN, care must be taken to see which side we are on. */ - if (x.s.hi == -0x1.0p63) { - - int64_t result = INT64_MIN; - - if (0.0 < x.s.lo) - { - /* If the tail is positive, the correct result is something other than INT64_MIN. - * we'll need to figure out what it is. - */ - - const doublebits lobits = { .d = x.s.lo }; - int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); - tailMantissa |= INT64_C(0x0010000000000000); - - /* Now we negate the tailMantissa */ - tailMantissa = (tailMantissa ^ INT64_C(-1)) + INT64_C(1); - - /* And shift it by the appropriate amount */ - const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; - tailMantissa >>= 1075 - biasedTailExponent; - - result -= tailMantissa; - } - - return result; - } - - /* Signed overflows, infinities, and NaNs */ - if (x.s.hi > 0.0) - return INT64_MAX; - else - return INT64_MIN; + // Signed overflows, infinities, and NaNs + if (x.s.hi > 0.0) + return INT64_MAX; + else + return INT64_MIN; } diff --git a/lib/builtins/ppc/fixunstfdi.c b/lib/builtins/ppc/fixunstfdi.c index 5e6e2cedf6ac..8d53f372527a 100644 --- a/lib/builtins/ppc/fixunstfdi.c +++ b/lib/builtins/ppc/fixunstfdi.c @@ -1,59 +1,57 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* uint64_t __fixunstfdi(long double x); */ -/* This file implements the PowerPC 128-bit double-double -> uint64_t conversion */ +// uint64_t __fixunstfdi(long double x); +// This file implements the PowerPC 128-bit double-double -> uint64_t conversion #include "DD.h" -uint64_t __fixunstfdi(long double input) -{ - const DD x = { .ld = input }; - const doublebits hibits = { .d = x.s.hi }; - - const uint32_t highWordMinusOne = (uint32_t)(hibits.x >> 32) - UINT32_C(0x3ff00000); - - /* If (1.0 - tiny) <= input < 0x1.0p64: */ - if (UINT32_C(0x04000000) > highWordMinusOne) - { - const int unbiasedHeadExponent = highWordMinusOne >> 20; - - uint64_t result = hibits.x & UINT64_C(0x000fffffffffffff); /* mantissa(hi) */ - result |= UINT64_C(0x0010000000000000); /* matissa(hi) with implicit bit */ - result <<= 11; /* mantissa(hi) left aligned in the int64 field. */ - - /* If the tail is non-zero, we need to patch in the tail bits. */ - if (0.0 != x.s.lo) - { - const doublebits lobits = { .d = x.s.lo }; - int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); - tailMantissa |= INT64_C(0x0010000000000000); - - /* At this point we have the mantissa of |tail| */ - - const int64_t negationMask = ((int64_t)(lobits.x)) >> 63; - tailMantissa = (tailMantissa ^ negationMask) - negationMask; - - /* Now we have the mantissa of tail as a signed 2s-complement integer */ - - const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; - - /* Shift the tail mantissa into the right position, accounting for the - * bias of 11 that we shifted the head mantissa by. - */ - tailMantissa >>= (unbiasedHeadExponent - (biasedTailExponent - (1023 - 11))); - - result += tailMantissa; - } - - result >>= (63 - unbiasedHeadExponent); - return result; - } - - /* Edge cases are handled here, with saturation. */ - if (1.0 > x.s.hi) - return UINT64_C(0); - else - return UINT64_MAX; +uint64_t __fixunstfdi(long double input) { + const DD x = {.ld = input}; + const doublebits hibits = {.d = x.s.hi}; + + const uint32_t highWordMinusOne = + (uint32_t)(hibits.x >> 32) - UINT32_C(0x3ff00000); + + // If (1.0 - tiny) <= input < 0x1.0p64: + if (UINT32_C(0x04000000) > highWordMinusOne) { + const int unbiasedHeadExponent = highWordMinusOne >> 20; + + uint64_t result = hibits.x & UINT64_C(0x000fffffffffffff); // mantissa(hi) + result |= UINT64_C(0x0010000000000000); // matissa(hi) with implicit bit + result <<= 11; // mantissa(hi) left aligned in the int64 field. + + // If the tail is non-zero, we need to patch in the tail bits. + if (0.0 != x.s.lo) { + const doublebits lobits = {.d = x.s.lo}; + int64_t tailMantissa = lobits.x & INT64_C(0x000fffffffffffff); + tailMantissa |= INT64_C(0x0010000000000000); + + // At this point we have the mantissa of |tail| + + const int64_t negationMask = ((int64_t)(lobits.x)) >> 63; + tailMantissa = (tailMantissa ^ negationMask) - negationMask; + + // Now we have the mantissa of tail as a signed 2s-complement integer + + const int biasedTailExponent = (int)(lobits.x >> 52) & 0x7ff; + + // Shift the tail mantissa into the right position, accounting for the + // bias of 11 that we shifted the head mantissa by. + tailMantissa >>= + (unbiasedHeadExponent - (biasedTailExponent - (1023 - 11))); + + result += tailMantissa; + } + + result >>= (63 - unbiasedHeadExponent); + return result; + } + + // Edge cases are handled here, with saturation. + if (1.0 > x.s.hi) + return UINT64_C(0); + else + return UINT64_MAX; } diff --git a/lib/builtins/ppc/fixunstfti.c b/lib/builtins/ppc/fixunstfti.c index fa21084cb576..1d19e01e3a91 100644 --- a/lib/builtins/ppc/fixunstfti.c +++ b/lib/builtins/ppc/fixunstfti.c @@ -1,9 +1,8 @@ //===-- lib/builtins/ppc/fixunstfti.c - Convert long double->int128 *-C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,10 +14,10 @@ #include "../int_math.h" #define BIAS 1023 -/* Convert long double into an unsigned 128-bit integer. */ +// Convert long double into an unsigned 128-bit integer. __uint128_t __fixunstfti(long double input) { - /* If we are trying to convert a NaN, return the NaN bit pattern. */ + // If we are trying to convert a NaN, return the NaN bit pattern. if (crt_isnan(input)) { return ((__uint128_t)0x7FF8000000000000ll) << 64 | (__uint128_t)0x0000000000000000ll; @@ -26,59 +25,59 @@ __uint128_t __fixunstfti(long double input) { __uint128_t result, hiResult, loResult; int hiExponent, loExponent, shift; - /* The long double representation, with the high and low portions of - * the long double, and the corresponding bit patterns of each double. */ + // The long double representation, with the high and low portions of + // the long double, and the corresponding bit patterns of each double. union { long double ld; - double d[2]; /* [0] is the high double, [1] is the low double. */ - unsigned long long ull[2]; /* High and low doubles as 64-bit integers. */ + double d[2]; // [0] is the high double, [1] is the low double. + unsigned long long ull[2]; // High and low doubles as 64-bit integers. } ldUnion; - /* If the long double is less than 1.0 or negative, - * return 0.0. */ + // If the long double is less than 1.0 or negative, + // return 0.0. if (input < 1.0) return 0.0; - /* Retrieve the 64-bit patterns of high and low doubles. - * Compute the unbiased exponent of both high and low doubles by - * removing the signs, isolating the exponent, and subtracting - * the bias from it. */ + // Retrieve the 64-bit patterns of high and low doubles. + // Compute the unbiased exponent of both high and low doubles by + // removing the signs, isolating the exponent, and subtracting + // the bias from it. ldUnion.ld = input; hiExponent = ((ldUnion.ull[0] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; loExponent = ((ldUnion.ull[1] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; - /* Convert each double into int64; they will be added to the int128 result. - * CASE 1: High or low double fits in int64 - * - Convert the each double normally into int64. - * - * CASE 2: High or low double does not fit in int64 - * - Scale the double to fit within a 64-bit integer - * - Calculate the shift (amount to scale the double by in the int128) - * - Clear all the bits of the exponent (with 0x800FFFFFFFFFFFFF) - * - Add BIAS+53 (0x4350000000000000) to exponent to correct the value - * - Scale (move) the double to the correct place in the int128 - * (Move it by 2^53 places) - * - * Note: If the high double is assumed to be positive, an unsigned conversion - * from long double to 64-bit integer is needed. The low double can be either - * positive or negative, so a signed conversion is needed to retain the result - * of the low double and to ensure it does not simply get converted to 0. */ + // Convert each double into int64; they will be added to the int128 result. + // CASE 1: High or low double fits in int64 + // - Convert the each double normally into int64. + // + // CASE 2: High or low double does not fit in int64 + // - Scale the double to fit within a 64-bit integer + // - Calculate the shift (amount to scale the double by in the int128) + // - Clear all the bits of the exponent (with 0x800FFFFFFFFFFFFF) + // - Add BIAS+53 (0x4350000000000000) to exponent to correct the value + // - Scale (move) the double to the correct place in the int128 + // (Move it by 2^53 places) + // + // Note: If the high double is assumed to be positive, an unsigned conversion + // from long double to 64-bit integer is needed. The low double can be either + // positive or negative, so a signed conversion is needed to retain the result + // of the low double and to ensure it does not simply get converted to 0. - /* CASE 1 - High double fits in int64. */ + // CASE 1 - High double fits in int64. if (hiExponent < 63) { hiResult = (unsigned long long)ldUnion.d[0]; } else if (hiExponent < 128) { - /* CASE 2 - High double does not fit in int64, scale and convert it. */ + // CASE 2 - High double does not fit in int64, scale and convert it. shift = hiExponent - 54; ldUnion.ull[0] &= 0x800FFFFFFFFFFFFFll; ldUnion.ull[0] |= 0x4350000000000000ll; hiResult = (unsigned long long)ldUnion.d[0]; hiResult <<= shift; } else { - /* Detect cases for overflow. When the exponent of the high - * double is greater than 128 bits and when the long double - * input is positive, return the max 128-bit integer. - * For negative inputs with exponents > 128, return 1, like gcc. */ + // Detect cases for overflow. When the exponent of the high + // double is greater than 128 bits and when the long double + // input is positive, return the max 128-bit integer. + // For negative inputs with exponents > 128, return 1, like gcc. if (ldUnion.d[0] > 0) { return ((__uint128_t)0xFFFFFFFFFFFFFFFFll) << 64 | (__uint128_t)0xFFFFFFFFFFFFFFFFll; @@ -88,11 +87,11 @@ __uint128_t __fixunstfti(long double input) { } } - /* CASE 1 - Low double fits in int64. */ + // CASE 1 - Low double fits in int64. if (loExponent < 63) { loResult = (long long)ldUnion.d[1]; } else { - /* CASE 2 - Low double does not fit in int64, scale and convert it. */ + // CASE 2 - Low double does not fit in int64, scale and convert it. shift = loExponent - 54; ldUnion.ull[1] &= 0x800FFFFFFFFFFFFFll; ldUnion.ull[1] |= 0x4350000000000000ll; @@ -100,7 +99,7 @@ __uint128_t __fixunstfti(long double input) { loResult <<= shift; } - /* Add the high and low doublewords together to form a 128 bit integer. */ + // Add the high and low doublewords together to form a 128 bit integer. result = loResult + hiResult; return result; } diff --git a/lib/builtins/ppc/floatditf.c b/lib/builtins/ppc/floatditf.c index beabdd017422..4c365418f082 100644 --- a/lib/builtins/ppc/floatditf.c +++ b/lib/builtins/ppc/floatditf.c @@ -1,36 +1,33 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __floatditf(long long x); */ -/* This file implements the PowerPC long long -> long double conversion */ +// long double __floatditf(long long x); +// This file implements the PowerPC long long -> long double conversion #include "DD.h" long double __floatditf(int64_t a) { - - static const double twop32 = 0x1.0p32; - static const double twop52 = 0x1.0p52; - - doublebits low = { .d = twop52 }; - low.x |= a & UINT64_C(0x00000000ffffffff); /* 0x1.0p52 + low 32 bits of a. */ - - const double high_addend = (double)((int32_t)(a >> 32))*twop32 - twop52; - - /* At this point, we have two double precision numbers - * high_addend and low.d, and we wish to return their sum - * as a canonicalized long double: - */ - - /* This implementation sets the inexact flag spuriously. - * This could be avoided, but at some substantial cost. - */ - - DD result; - - result.s.hi = high_addend + low.d; - result.s.lo = (high_addend - result.s.hi) + low.d; - - return result.ld; - + + static const double twop32 = 0x1.0p32; + static const double twop52 = 0x1.0p52; + + doublebits low = {.d = twop52}; + low.x |= a & UINT64_C(0x00000000ffffffff); // 0x1.0p52 + low 32 bits of a. + + const double high_addend = (double)((int32_t)(a >> 32)) * twop32 - twop52; + + // At this point, we have two double precision numbers + // high_addend and low.d, and we wish to return their sum + // as a canonicalized long double: + + // This implementation sets the inexact flag spuriously. + // This could be avoided, but at some substantial cost. + + DD result; + + result.s.hi = high_addend + low.d; + result.s.lo = (high_addend - result.s.hi) + low.d; + + return result.ld; } diff --git a/lib/builtins/ppc/floattitf.c b/lib/builtins/ppc/floattitf.c index b8e297b6b8d9..6deac6498128 100644 --- a/lib/builtins/ppc/floattitf.c +++ b/lib/builtins/ppc/floattitf.c @@ -1,9 +1,8 @@ //===-- lib/builtins/ppc/floattitf.c - Convert int128->long double -*-C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,35 +13,34 @@ #include <stdint.h> -/* Conversions from signed and unsigned 64-bit int to long double. */ +// Conversions from signed and unsigned 64-bit int to long double. long double __floatditf(int64_t); long double __floatunditf(uint64_t); -/* Convert a signed 128-bit integer to long double. - * This uses the following property: Let hi and lo be 64-bits each, - * and let signed_val_k() and unsigned_val_k() be the value of the - * argument interpreted as a signed or unsigned k-bit integer. Then, - * - * signed_val_128(hi,lo) = signed_val_64(hi) * 2^64 + unsigned_val_64(lo) - * = (long double)hi * 2^64 + (long double)lo, - * - * where (long double)hi and (long double)lo are signed and - * unsigned 64-bit integer to long double conversions, respectively. - */ +// Convert a signed 128-bit integer to long double. +// This uses the following property: Let hi and lo be 64-bits each, +// and let signed_val_k() and unsigned_val_k() be the value of the +// argument interpreted as a signed or unsigned k-bit integer. Then, +// +// signed_val_128(hi,lo) = signed_val_64(hi) * 2^64 + unsigned_val_64(lo) +// = (long double)hi * 2^64 + (long double)lo, +// +// where (long double)hi and (long double)lo are signed and +// unsigned 64-bit integer to long double conversions, respectively. long double __floattitf(__int128_t arg) { - /* Split the int128 argument into 64-bit high and low int64 parts. */ + // Split the int128 argument into 64-bit high and low int64 parts. int64_t ArgHiPart = (int64_t)(arg >> 64); uint64_t ArgLoPart = (uint64_t)arg; - /* Convert each 64-bit part into long double. The high part - * must be a signed conversion and the low part an unsigned conversion - * to ensure the correct result. */ + // Convert each 64-bit part into long double. The high part + // must be a signed conversion and the low part an unsigned conversion + // to ensure the correct result. long double ConvertedHiPart = __floatditf(ArgHiPart); long double ConvertedLoPart = __floatunditf(ArgLoPart); - /* The low bit of ArgHiPart corresponds to the 2^64 bit in arg. - * Multiply the high part by 2^64 to undo the right shift by 64-bits - * done in the splitting. Then, add to the low part to obtain the - * final result. */ + // The low bit of ArgHiPart corresponds to the 2^64 bit in arg. + // Multiply the high part by 2^64 to undo the right shift by 64-bits + // done in the splitting. Then, add to the low part to obtain the + // final result. return ((ConvertedHiPart * 0x1.0p64) + ConvertedLoPart); } diff --git a/lib/builtins/ppc/floatunditf.c b/lib/builtins/ppc/floatunditf.c index b12e1e738fd0..fb4cd3f91d86 100644 --- a/lib/builtins/ppc/floatunditf.c +++ b/lib/builtins/ppc/floatunditf.c @@ -1,41 +1,39 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __floatunditf(unsigned long long x); */ -/* This file implements the PowerPC unsigned long long -> long double conversion */ +// long double __floatunditf(unsigned long long x); +// This file implements the PowerPC unsigned long long -> long double conversion #include "DD.h" long double __floatunditf(uint64_t a) { - - /* Begins with an exact copy of the code from __floatundidf */ - - static const double twop52 = 0x1.0p52; - static const double twop84 = 0x1.0p84; - static const double twop84_plus_twop52 = 0x1.00000001p84; - - doublebits high = { .d = twop84 }; - doublebits low = { .d = twop52 }; - - high.x |= a >> 32; /* 0x1.0p84 + high 32 bits of a */ - low.x |= a & UINT64_C(0x00000000ffffffff); /* 0x1.0p52 + low 32 bits of a */ - - const double high_addend = high.d - twop84_plus_twop52; - - /* At this point, we have two double precision numbers - * high_addend and low.d, and we wish to return their sum - * as a canonicalized long double: - */ - - /* This implementation sets the inexact flag spuriously. */ - /* This could be avoided, but at some substantial cost. */ - - DD result; - - result.s.hi = high_addend + low.d; - result.s.lo = (high_addend - result.s.hi) + low.d; - - return result.ld; - + + // Begins with an exact copy of the code from __floatundidf + + static const double twop52 = 0x1.0p52; + static const double twop84 = 0x1.0p84; + static const double twop84_plus_twop52 = 0x1.00000001p84; + + doublebits high = {.d = twop84}; + doublebits low = {.d = twop52}; + + high.x |= a >> 32; // 0x1.0p84 + high 32 bits of a + low.x |= a & UINT64_C(0x00000000ffffffff); // 0x1.0p52 + low 32 bits of a + + const double high_addend = high.d - twop84_plus_twop52; + + // At this point, we have two double precision numbers + // high_addend and low.d, and we wish to return their sum + // as a canonicalized long double: + + // This implementation sets the inexact flag spuriously. + // This could be avoided, but at some substantial cost. + + DD result; + + result.s.hi = high_addend + low.d; + result.s.lo = (high_addend - result.s.hi) + low.d; + + return result.ld; } diff --git a/lib/builtins/ppc/gcc_qadd.c b/lib/builtins/ppc/gcc_qadd.c index 32e16e9d1d11..6e1e63cb530e 100644 --- a/lib/builtins/ppc/gcc_qadd.c +++ b/lib/builtins/ppc/gcc_qadd.c @@ -1,76 +1,74 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qadd(long double x, long double y); - * This file implements the PowerPC 128-bit double-double add operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qadd(long double x, long double y); +// This file implements the PowerPC 128-bit double-double add operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qadd(long double x, long double y) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - - DD dst = { .ld = x }, src = { .ld = y }; - - register double A = dst.s.hi, a = dst.s.lo, - B = src.s.hi, b = src.s.lo; - - /* If both operands are zero: */ - if ((A == 0.0) && (B == 0.0)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If either operand is NaN or infinity: */ - const doublebits abits = { .d = A }; - const doublebits bbits = { .d = B }; - if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || - (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If the computation overflows: */ - /* This may be playing things a little bit fast and loose, but it will do for a start. */ - const double testForOverflow = A + (B + (a + b)); - const doublebits testbits = { .d = testForOverflow }; - if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = testForOverflow; - dst.s.lo = 0.0; - return dst.ld; - } - - double H, h; - double T, t; - double W, w; - double Y; - - H = B + (A - (A + B)); - T = b + (a - (a + b)); - h = A + (B - (A + B)); - t = a + (b - (a + b)); - - if (local_fabs(A) <= local_fabs(B)) - w = (a + b) + h; - else - w = (a + b) + H; - - W = (A + B) + w; - Y = (A + B) - W; - Y += w; - - if (local_fabs(a) <= local_fabs(b)) - w = t + Y; - else - w = T + Y; - - dst.s.hi = Y = W + w; - dst.s.lo = (W - Y) + w; - - return dst.ld; +long double __gcc_qadd(long double x, long double y) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + + DD dst = {.ld = x}, src = {.ld = y}; + + register double A = dst.s.hi, a = dst.s.lo, B = src.s.hi, b = src.s.lo; + + // If both operands are zero: + if ((A == 0.0) && (B == 0.0)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If either operand is NaN or infinity: + const doublebits abits = {.d = A}; + const doublebits bbits = {.d = B}; + if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || + (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If the computation overflows: + // This may be playing things a little bit fast and loose, but it will do for + // a start. + const double testForOverflow = A + (B + (a + b)); + const doublebits testbits = {.d = testForOverflow}; + if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = testForOverflow; + dst.s.lo = 0.0; + return dst.ld; + } + + double H, h; + double T, t; + double W, w; + double Y; + + H = B + (A - (A + B)); + T = b + (a - (a + b)); + h = A + (B - (A + B)); + t = a + (b - (a + b)); + + if (local_fabs(A) <= local_fabs(B)) + w = (a + b) + h; + else + w = (a + b) + H; + + W = (A + B) + w; + Y = (A + B) - W; + Y += w; + + if (local_fabs(a) <= local_fabs(b)) + w = t + Y; + else + w = T + Y; + + dst.s.hi = Y = W + w; + dst.s.lo = (W - Y) + w; + + return dst.ld; } diff --git a/lib/builtins/ppc/gcc_qdiv.c b/lib/builtins/ppc/gcc_qdiv.c index 70aa00b64400..35a3cbc3d3f8 100644 --- a/lib/builtins/ppc/gcc_qdiv.c +++ b/lib/builtins/ppc/gcc_qdiv.c @@ -1,55 +1,52 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qdiv(long double x, long double y); - * This file implements the PowerPC 128-bit double-double division operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qdiv(long double x, long double y); +// This file implements the PowerPC 128-bit double-double division operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qdiv(long double a, long double b) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - DD dst = { .ld = a }, src = { .ld = b }; - - register double x = dst.s.hi, x1 = dst.s.lo, - y = src.s.hi, y1 = src.s.lo; - - double yHi, yLo, qHi, qLo; - double yq, tmp, q; - - q = x / y; - - /* Detect special cases */ - if (q == 0.0) { - dst.s.hi = q; - dst.s.lo = 0.0; - return dst.ld; - } - - const doublebits qBits = { .d = q }; - if (((uint32_t)(qBits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = q; - dst.s.lo = 0.0; - return dst.ld; - } - - yHi = high26bits(y); - qHi = high26bits(q); - - yq = y * q; - yLo = y - yHi; - qLo = q - qHi; - - tmp = LOWORDER(yq, yHi, yLo, qHi, qLo); - tmp = (x - yq) - tmp; - tmp = ((tmp + x1) - y1 * q) / y; - x = q + tmp; - - dst.s.lo = (q - x) + tmp; - dst.s.hi = x; - +long double __gcc_qdiv(long double a, long double b) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + DD dst = {.ld = a}, src = {.ld = b}; + + register double x = dst.s.hi, x1 = dst.s.lo, y = src.s.hi, y1 = src.s.lo; + + double yHi, yLo, qHi, qLo; + double yq, tmp, q; + + q = x / y; + + // Detect special cases + if (q == 0.0) { + dst.s.hi = q; + dst.s.lo = 0.0; + return dst.ld; + } + + const doublebits qBits = {.d = q}; + if (((uint32_t)(qBits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = q; + dst.s.lo = 0.0; return dst.ld; + } + + yHi = high26bits(y); + qHi = high26bits(q); + + yq = y * q; + yLo = y - yHi; + qLo = q - qHi; + + tmp = LOWORDER(yq, yHi, yLo, qHi, qLo); + tmp = (x - yq) - tmp; + tmp = ((tmp + x1) - y1 * q) / y; + x = q + tmp; + + dst.s.lo = (q - x) + tmp; + dst.s.hi = x; + + return dst.ld; } diff --git a/lib/builtins/ppc/gcc_qmul.c b/lib/builtins/ppc/gcc_qmul.c index fb4c5164ccb5..75f519aad6f0 100644 --- a/lib/builtins/ppc/gcc_qmul.c +++ b/lib/builtins/ppc/gcc_qmul.c @@ -1,53 +1,50 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qmul(long double x, long double y); - * This file implements the PowerPC 128-bit double-double multiply operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qmul(long double x, long double y); +// This file implements the PowerPC 128-bit double-double multiply operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qmul(long double x, long double y) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - DD dst = { .ld = x }, src = { .ld = y }; - - register double A = dst.s.hi, a = dst.s.lo, - B = src.s.hi, b = src.s.lo; - - double aHi, aLo, bHi, bLo; - double ab, tmp, tau; - - ab = A * B; - - /* Detect special cases */ - if (ab == 0.0) { - dst.s.hi = ab; - dst.s.lo = 0.0; - return dst.ld; - } - - const doublebits abBits = { .d = ab }; - if (((uint32_t)(abBits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = ab; - dst.s.lo = 0.0; - return dst.ld; - } - - /* Generic cases handled here. */ - aHi = high26bits(A); - bHi = high26bits(B); - aLo = A - aHi; - bLo = B - bHi; - - tmp = LOWORDER(ab, aHi, aLo, bHi, bLo); - tmp += (A * b + a * B); - tau = ab + tmp; - - dst.s.lo = (ab - tau) + tmp; - dst.s.hi = tau; - +long double __gcc_qmul(long double x, long double y) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + DD dst = {.ld = x}, src = {.ld = y}; + + register double A = dst.s.hi, a = dst.s.lo, B = src.s.hi, b = src.s.lo; + + double aHi, aLo, bHi, bLo; + double ab, tmp, tau; + + ab = A * B; + + // Detect special cases + if (ab == 0.0) { + dst.s.hi = ab; + dst.s.lo = 0.0; + return dst.ld; + } + + const doublebits abBits = {.d = ab}; + if (((uint32_t)(abBits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = ab; + dst.s.lo = 0.0; return dst.ld; + } + + // Generic cases handled here. + aHi = high26bits(A); + bHi = high26bits(B); + aLo = A - aHi; + bLo = B - bHi; + + tmp = LOWORDER(ab, aHi, aLo, bHi, bLo); + tmp += (A * b + a * B); + tau = ab + tmp; + + dst.s.lo = (ab - tau) + tmp; + dst.s.hi = tau; + + return dst.ld; } diff --git a/lib/builtins/ppc/gcc_qsub.c b/lib/builtins/ppc/gcc_qsub.c index c092e24dbda1..ac08120be0bd 100644 --- a/lib/builtins/ppc/gcc_qsub.c +++ b/lib/builtins/ppc/gcc_qsub.c @@ -1,76 +1,74 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __gcc_qsub(long double x, long double y); - * This file implements the PowerPC 128-bit double-double add operation. - * This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) - */ +// long double __gcc_qsub(long double x, long double y); +// This file implements the PowerPC 128-bit double-double add operation. +// This implementation is shamelessly cribbed from Apple's DDRT, circa 1993(!) #include "DD.h" -long double __gcc_qsub(long double x, long double y) -{ - static const uint32_t infinityHi = UINT32_C(0x7ff00000); - - DD dst = { .ld = x }, src = { .ld = y }; - - register double A = dst.s.hi, a = dst.s.lo, - B = -src.s.hi, b = -src.s.lo; - - /* If both operands are zero: */ - if ((A == 0.0) && (B == 0.0)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If either operand is NaN or infinity: */ - const doublebits abits = { .d = A }; - const doublebits bbits = { .d = B }; - if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || - (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { - dst.s.hi = A + B; - dst.s.lo = 0.0; - return dst.ld; - } - - /* If the computation overflows: */ - /* This may be playing things a little bit fast and loose, but it will do for a start. */ - const double testForOverflow = A + (B + (a + b)); - const doublebits testbits = { .d = testForOverflow }; - if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { - dst.s.hi = testForOverflow; - dst.s.lo = 0.0; - return dst.ld; - } - - double H, h; - double T, t; - double W, w; - double Y; - - H = B + (A - (A + B)); - T = b + (a - (a + b)); - h = A + (B - (A + B)); - t = a + (b - (a + b)); - - if (local_fabs(A) <= local_fabs(B)) - w = (a + b) + h; - else - w = (a + b) + H; - - W = (A + B) + w; - Y = (A + B) - W; - Y += w; - - if (local_fabs(a) <= local_fabs(b)) - w = t + Y; - else - w = T + Y; - - dst.s.hi = Y = W + w; - dst.s.lo = (W - Y) + w; - - return dst.ld; +long double __gcc_qsub(long double x, long double y) { + static const uint32_t infinityHi = UINT32_C(0x7ff00000); + + DD dst = {.ld = x}, src = {.ld = y}; + + register double A = dst.s.hi, a = dst.s.lo, B = -src.s.hi, b = -src.s.lo; + + // If both operands are zero: + if ((A == 0.0) && (B == 0.0)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If either operand is NaN or infinity: + const doublebits abits = {.d = A}; + const doublebits bbits = {.d = B}; + if ((((uint32_t)(abits.x >> 32) & infinityHi) == infinityHi) || + (((uint32_t)(bbits.x >> 32) & infinityHi) == infinityHi)) { + dst.s.hi = A + B; + dst.s.lo = 0.0; + return dst.ld; + } + + // If the computation overflows: + // This may be playing things a little bit fast and loose, but it will do for + // a start. + const double testForOverflow = A + (B + (a + b)); + const doublebits testbits = {.d = testForOverflow}; + if (((uint32_t)(testbits.x >> 32) & infinityHi) == infinityHi) { + dst.s.hi = testForOverflow; + dst.s.lo = 0.0; + return dst.ld; + } + + double H, h; + double T, t; + double W, w; + double Y; + + H = B + (A - (A + B)); + T = b + (a - (a + b)); + h = A + (B - (A + B)); + t = a + (b - (a + b)); + + if (local_fabs(A) <= local_fabs(B)) + w = (a + b) + h; + else + w = (a + b) + H; + + W = (A + B) + w; + Y = (A + B) - W; + Y += w; + + if (local_fabs(a) <= local_fabs(b)) + w = t + Y; + else + w = T + Y; + + dst.s.hi = Y = W + w; + dst.s.lo = (W - Y) + w; + + return dst.ld; } diff --git a/lib/builtins/ppc/multc3.c b/lib/builtins/ppc/multc3.c index 9dd79c975dde..f1fd6816d6c8 100644 --- a/lib/builtins/ppc/multc3.c +++ b/lib/builtins/ppc/multc3.c @@ -1,90 +1,85 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "DD.h" #include "../int_math.h" +#include "DD.h" -#define makeFinite(x) { \ - (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ - (x).s.lo = 0.0; \ +#define makeFinite(x) \ + { \ + (x).s.hi = crt_copysign(crt_isinf((x).s.hi) ? 1.0 : 0.0, (x).s.hi); \ + (x).s.lo = 0.0; \ } -#define zeroNaN(x) { \ - if (crt_isnan((x).s.hi)) { \ - (x).s.hi = crt_copysign(0.0, (x).s.hi); \ - (x).s.lo = 0.0; \ - } \ +#define zeroNaN(x) \ + { \ + if (crt_isnan((x).s.hi)) { \ + (x).s.hi = crt_copysign(0.0, (x).s.hi); \ + (x).s.lo = 0.0; \ + } \ } -long double _Complex -__multc3(long double a, long double b, long double c, long double d) -{ - long double ac = __gcc_qmul(a,c); - long double bd = __gcc_qmul(b,d); - long double ad = __gcc_qmul(a,d); - long double bc = __gcc_qmul(b,c); - - DD real = { .ld = __gcc_qsub(ac,bd) }; - DD imag = { .ld = __gcc_qadd(ad,bc) }; - - if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) - { - int recalc = 0; - - DD aDD = { .ld = a }; - DD bDD = { .ld = b }; - DD cDD = { .ld = c }; - DD dDD = { .ld = d }; - - if (crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) - { - makeFinite(aDD); - makeFinite(bDD); - zeroNaN(cDD); - zeroNaN(dDD); - recalc = 1; - } - - if (crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) - { - makeFinite(cDD); - makeFinite(dDD); - zeroNaN(aDD); - zeroNaN(bDD); - recalc = 1; - } - - if (!recalc) - { - DD acDD = { .ld = ac }; - DD bdDD = { .ld = bd }; - DD adDD = { .ld = ad }; - DD bcDD = { .ld = bc }; - - if (crt_isinf(acDD.s.hi) || crt_isinf(bdDD.s.hi) || - crt_isinf(adDD.s.hi) || crt_isinf(bcDD.s.hi)) - { - zeroNaN(aDD); - zeroNaN(bDD); - zeroNaN(cDD); - zeroNaN(dDD); - recalc = 1; - } - } - - if (recalc) - { - real.s.hi = CRT_INFINITY * (aDD.s.hi*cDD.s.hi - bDD.s.hi*dDD.s.hi); - real.s.lo = 0.0; - imag.s.hi = CRT_INFINITY * (aDD.s.hi*dDD.s.hi + bDD.s.hi*cDD.s.hi); - imag.s.lo = 0.0; - } - } - - long double _Complex z; - __real__ z = real.ld; - __imag__ z = imag.ld; - - return z; +long double _Complex __multc3(long double a, long double b, long double c, + long double d) { + long double ac = __gcc_qmul(a, c); + long double bd = __gcc_qmul(b, d); + long double ad = __gcc_qmul(a, d); + long double bc = __gcc_qmul(b, c); + + DD real = {.ld = __gcc_qsub(ac, bd)}; + DD imag = {.ld = __gcc_qadd(ad, bc)}; + + if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) { + int recalc = 0; + + DD aDD = {.ld = a}; + DD bDD = {.ld = b}; + DD cDD = {.ld = c}; + DD dDD = {.ld = d}; + + if (crt_isinf(aDD.s.hi) || crt_isinf(bDD.s.hi)) { + makeFinite(aDD); + makeFinite(bDD); + zeroNaN(cDD); + zeroNaN(dDD); + recalc = 1; + } + + if (crt_isinf(cDD.s.hi) || crt_isinf(dDD.s.hi)) { + makeFinite(cDD); + makeFinite(dDD); + zeroNaN(aDD); + zeroNaN(bDD); + recalc = 1; + } + + if (!recalc) { + DD acDD = {.ld = ac}; + DD bdDD = {.ld = bd}; + DD adDD = {.ld = ad}; + DD bcDD = {.ld = bc}; + + if (crt_isinf(acDD.s.hi) || crt_isinf(bdDD.s.hi) || + crt_isinf(adDD.s.hi) || crt_isinf(bcDD.s.hi)) { + zeroNaN(aDD); + zeroNaN(bDD); + zeroNaN(cDD); + zeroNaN(dDD); + recalc = 1; + } + } + + if (recalc) { + real.s.hi = CRT_INFINITY * (aDD.s.hi * cDD.s.hi - bDD.s.hi * dDD.s.hi); + real.s.lo = 0.0; + imag.s.hi = CRT_INFINITY * (aDD.s.hi * dDD.s.hi + bDD.s.hi * cDD.s.hi); + imag.s.lo = 0.0; + } + } + + long double _Complex z; + __real__ z = real.ld; + __imag__ z = imag.ld; + + return z; } diff --git a/lib/builtins/ppc/restFP.S b/lib/builtins/ppc/restFP.S index 507e756e18b1..02317bd6a649 100644 --- a/lib/builtins/ppc/restFP.S +++ b/lib/builtins/ppc/restFP.S @@ -1,9 +1,8 @@ //===-- restFP.S - Implement restFP ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/ppc/saveFP.S b/lib/builtins/ppc/saveFP.S index 20b06fff53d9..1ef5532c8a83 100644 --- a/lib/builtins/ppc/saveFP.S +++ b/lib/builtins/ppc/saveFP.S @@ -1,9 +1,8 @@ //===-- saveFP.S - Implement saveFP ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/riscv/mulsi3.S b/lib/builtins/riscv/mulsi3.S index a58d237040b6..5464919b26b9 100644 --- a/lib/builtins/riscv/mulsi3.S +++ b/lib/builtins/riscv/mulsi3.S @@ -1,9 +1,8 @@ //===--- mulsi3.S - Integer multiplication routines routines ---===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/builtins/subdf3.c b/lib/builtins/subdf3.c index a892fa603cf2..5346dbc970fa 100644 --- a/lib/builtins/subdf3.c +++ b/lib/builtins/subdf3.c @@ -1,9 +1,8 @@ //===-- lib/adddf3.c - Double-precision subtraction ---------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,17 +15,14 @@ #include "fp_lib.h" // Subtraction; flip the sign bit of b and add. -COMPILER_RT_ABI fp_t -__subdf3(fp_t a, fp_t b) { - return __adddf3(a, fromRep(toRep(b) ^ signBit)); +COMPILER_RT_ABI fp_t __subdf3(fp_t a, fp_t b) { + return __adddf3(a, fromRep(toRep(b) ^ signBit)); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) { - return __subdf3(a, b); -} +AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) { return __subdf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subdf3); +COMPILER_RT_ALIAS(__subdf3, __aeabi_dsub) #endif #endif diff --git a/lib/builtins/subsf3.c b/lib/builtins/subsf3.c index 4b2786177dcc..85bde029b5bc 100644 --- a/lib/builtins/subsf3.c +++ b/lib/builtins/subsf3.c @@ -1,9 +1,8 @@ //===-- lib/subsf3.c - Single-precision subtraction ---------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,17 +15,14 @@ #include "fp_lib.h" // Subtraction; flip the sign bit of b and add. -COMPILER_RT_ABI fp_t -__subsf3(fp_t a, fp_t b) { - return __addsf3(a, fromRep(toRep(b) ^ signBit)); +COMPILER_RT_ABI fp_t __subsf3(fp_t a, fp_t b) { + return __addsf3(a, fromRep(toRep(b) ^ signBit)); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) { - return __subsf3(a, b); -} +AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) { return __subsf3(a, b); } #else -AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subsf3); +COMPILER_RT_ALIAS(__subsf3, __aeabi_fsub) #endif #endif diff --git a/lib/builtins/subtf3.c b/lib/builtins/subtf3.c index 609b816f41e3..c96814692d2c 100644 --- a/lib/builtins/subtf3.c +++ b/lib/builtins/subtf3.c @@ -1,9 +1,8 @@ //===-- lib/subtf3.c - Quad-precision subtraction -----------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,9 +18,8 @@ COMPILER_RT_ABI fp_t __addtf3(fp_t a, fp_t b); // Subtraction; flip the sign bit of b and add. -COMPILER_RT_ABI fp_t -__subtf3(fp_t a, fp_t b) { - return __addtf3(a, fromRep(toRep(b) ^ signBit)); +COMPILER_RT_ABI fp_t __subtf3(fp_t a, fp_t b) { + return __addtf3(a, fromRep(toRep(b) ^ signBit)); } #endif diff --git a/lib/builtins/subvdi3.c b/lib/builtins/subvdi3.c index 71fc70ffa92d..d7d78f1bb34f 100644 --- a/lib/builtins/subvdi3.c +++ b/lib/builtins/subvdi3.c @@ -1,36 +1,29 @@ -/* ===-- subvdi3.c - Implement __subvdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __subvdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- subvdi3.c - Implement __subvdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __subvdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a - b */ +// Returns: a - b -/* Effects: aborts if a - b overflows */ +// Effects: aborts if a - b overflows -COMPILER_RT_ABI di_int -__subvdi3(di_int a, di_int b) -{ - di_int s = (du_int) a - (du_int) b; - if (b >= 0) - { - if (s > a) - compilerrt_abort(); - } - else - { - if (s <= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI di_int __subvdi3(di_int a, di_int b) { + di_int s = (du_int)a - (du_int)b; + if (b >= 0) { + if (s > a) + compilerrt_abort(); + } else { + if (s <= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/subvsi3.c b/lib/builtins/subvsi3.c index e6c0fb688c97..c3cb6e8a217c 100644 --- a/lib/builtins/subvsi3.c +++ b/lib/builtins/subvsi3.c @@ -1,36 +1,29 @@ -/* ===-- subvsi3.c - Implement __subvsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __subvsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- subvsi3.c - Implement __subvsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __subvsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a - b */ +// Returns: a - b -/* Effects: aborts if a - b overflows */ +// Effects: aborts if a - b overflows -COMPILER_RT_ABI si_int -__subvsi3(si_int a, si_int b) -{ - si_int s = (su_int) a - (su_int) b; - if (b >= 0) - { - if (s > a) - compilerrt_abort(); - } - else - { - if (s <= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI si_int __subvsi3(si_int a, si_int b) { + si_int s = (su_int)a - (su_int)b; + if (b >= 0) { + if (s > a) + compilerrt_abort(); + } else { + if (s <= a) + compilerrt_abort(); + } + return s; } diff --git a/lib/builtins/subvti3.c b/lib/builtins/subvti3.c index a6804d2d7b95..91ac18834043 100644 --- a/lib/builtins/subvti3.c +++ b/lib/builtins/subvti3.c @@ -1,40 +1,33 @@ -/* ===-- subvti3.c - Implement __subvti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __subvti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- subvti3.c - Implement __subvti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __subvti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a - b */ +// Returns: a - b -/* Effects: aborts if a - b overflows */ +// Effects: aborts if a - b overflows -COMPILER_RT_ABI ti_int -__subvti3(ti_int a, ti_int b) -{ - ti_int s = (tu_int) a - (tu_int) b; - if (b >= 0) - { - if (s > a) - compilerrt_abort(); - } - else - { - if (s <= a) - compilerrt_abort(); - } - return s; +COMPILER_RT_ABI ti_int __subvti3(ti_int a, ti_int b) { + ti_int s = (tu_int)a - (tu_int)b; + if (b >= 0) { + if (s > a) + compilerrt_abort(); + } else { + if (s <= a) + compilerrt_abort(); + } + return s; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/trampoline_setup.c b/lib/builtins/trampoline_setup.c index 25b627ab7658..a62431723d78 100644 --- a/lib/builtins/trampoline_setup.c +++ b/lib/builtins/trampoline_setup.c @@ -1,48 +1,43 @@ -/* ===----- trampoline_setup.c - Implement __trampoline_setup -------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - */ +//===----- trampoline_setup.c - Implement __trampoline_setup -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -extern void __clear_cache(void* start, void* end); +extern void __clear_cache(void *start, void *end); -/* - * The ppc compiler generates calls to __trampoline_setup() when creating - * trampoline functions on the stack for use with nested functions. - * This function creates a custom 40-byte trampoline function on the stack - * which loads r11 with a pointer to the outer function's locals - * and then jumps to the target nested function. - */ +// The ppc compiler generates calls to __trampoline_setup() when creating +// trampoline functions on the stack for use with nested functions. +// This function creates a custom 40-byte trampoline function on the stack +// which loads r11 with a pointer to the outer function's locals +// and then jumps to the target nested function. #if __ppc__ && !defined(__powerpc64__) -COMPILER_RT_ABI void -__trampoline_setup(uint32_t* trampOnStack, int trampSizeAllocated, - const void* realFunc, void* localsPtr) -{ - /* should never happen, but if compiler did not allocate */ - /* enough space on stack for the trampoline, abort */ - if ( trampSizeAllocated < 40 ) - compilerrt_abort(); - - /* create trampoline */ - trampOnStack[0] = 0x7c0802a6; /* mflr r0 */ - trampOnStack[1] = 0x4800000d; /* bl Lbase */ - trampOnStack[2] = (uint32_t)realFunc; - trampOnStack[3] = (uint32_t)localsPtr; - trampOnStack[4] = 0x7d6802a6; /* Lbase: mflr r11 */ - trampOnStack[5] = 0x818b0000; /* lwz r12,0(r11) */ - trampOnStack[6] = 0x7c0803a6; /* mtlr r0 */ - trampOnStack[7] = 0x7d8903a6; /* mtctr r12 */ - trampOnStack[8] = 0x816b0004; /* lwz r11,4(r11) */ - trampOnStack[9] = 0x4e800420; /* bctr */ - - /* clear instruction cache */ - __clear_cache(trampOnStack, &trampOnStack[10]); +COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack, + int trampSizeAllocated, + const void *realFunc, void *localsPtr) { + // should never happen, but if compiler did not allocate + // enough space on stack for the trampoline, abort + if (trampSizeAllocated < 40) + compilerrt_abort(); + + // create trampoline + trampOnStack[0] = 0x7c0802a6; // mflr r0 + trampOnStack[1] = 0x4800000d; // bl Lbase + trampOnStack[2] = (uint32_t)realFunc; + trampOnStack[3] = (uint32_t)localsPtr; + trampOnStack[4] = 0x7d6802a6; // Lbase: mflr r11 + trampOnStack[5] = 0x818b0000; // lwz r12,0(r11) + trampOnStack[6] = 0x7c0803a6; // mtlr r0 + trampOnStack[7] = 0x7d8903a6; // mtctr r12 + trampOnStack[8] = 0x816b0004; // lwz r11,4(r11) + trampOnStack[9] = 0x4e800420; // bctr + + // clear instruction cache + __clear_cache(trampOnStack, &trampOnStack[10]); } -#endif /* __ppc__ && !defined(__powerpc64__) */ +#endif // __ppc__ && !defined(__powerpc64__) diff --git a/lib/builtins/truncdfhf2.c b/lib/builtins/truncdfhf2.c index 8354a41b8b6f..90c418a4387f 100644 --- a/lib/builtins/truncdfhf2.c +++ b/lib/builtins/truncdfhf2.c @@ -1,9 +1,8 @@ //===-- lib/truncdfhf2.c - double -> half conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,16 +10,12 @@ #define DST_HALF #include "fp_trunc_impl.inc" -COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { return __truncXfYf2__(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI uint16_t __aeabi_d2h(double a) { - return __truncdfhf2(a); -} +AEABI_RTABI uint16_t __aeabi_d2h(double a) { return __truncdfhf2(a); } #else -AEABI_RTABI uint16_t __aeabi_d2h(double a) COMPILER_RT_ALIAS(__truncdfhf2); +COMPILER_RT_ALIAS(__truncdfhf2, __aeabi_d2h) #endif #endif diff --git a/lib/builtins/truncdfsf2.c b/lib/builtins/truncdfsf2.c index 195d3e0656e7..44a1299e30ae 100644 --- a/lib/builtins/truncdfsf2.c +++ b/lib/builtins/truncdfsf2.c @@ -1,9 +1,8 @@ //===-- lib/truncdfsf2.c - double -> single conversion ------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,16 +10,12 @@ #define DST_SINGLE #include "fp_trunc_impl.inc" -COMPILER_RT_ABI float __truncdfsf2(double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI float __truncdfsf2(double a) { return __truncXfYf2__(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI float __aeabi_d2f(double a) { - return __truncdfsf2(a); -} +AEABI_RTABI float __aeabi_d2f(double a) { return __truncdfsf2(a); } #else -AEABI_RTABI float __aeabi_d2f(double a) COMPILER_RT_ALIAS(__truncdfsf2); +COMPILER_RT_ALIAS(__truncdfsf2, __aeabi_d2f) #endif #endif diff --git a/lib/builtins/truncsfhf2.c b/lib/builtins/truncsfhf2.c index 9c84ab4f938a..1f17194c38e5 100644 --- a/lib/builtins/truncsfhf2.c +++ b/lib/builtins/truncsfhf2.c @@ -1,9 +1,8 @@ //===-- lib/truncsfhf2.c - single -> half conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,19 +13,15 @@ // Use a forwarding definition and noinline to implement a poor man's alias, // as there isn't a good cross-platform way of defining one. COMPILER_RT_ABI NOINLINE uint16_t __truncsfhf2(float a) { - return __truncXfYf2__(a); + return __truncXfYf2__(a); } -COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { - return __truncsfhf2(a); -} +COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); } #if defined(__ARM_EABI__) #if defined(COMPILER_RT_ARMHF_TARGET) -AEABI_RTABI uint16_t __aeabi_f2h(float a) { - return __truncsfhf2(a); -} +AEABI_RTABI uint16_t __aeabi_f2h(float a) { return __truncsfhf2(a); } #else -AEABI_RTABI uint16_t __aeabi_f2h(float a) COMPILER_RT_ALIAS(__truncsfhf2); +COMPILER_RT_ALIAS(__truncsfhf2, __aeabi_f2h) #endif #endif diff --git a/lib/builtins/trunctfdf2.c b/lib/builtins/trunctfdf2.c index 741a71b33c5a..6857ea54d8a5 100644 --- a/lib/builtins/trunctfdf2.c +++ b/lib/builtins/trunctfdf2.c @@ -1,9 +1,8 @@ //===-- lib/truncdfsf2.c - quad -> double conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,8 +14,6 @@ #define DST_DOUBLE #include "fp_trunc_impl.inc" -COMPILER_RT_ABI double __trunctfdf2(long double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI double __trunctfdf2(long double a) { return __truncXfYf2__(a); } #endif diff --git a/lib/builtins/trunctfsf2.c b/lib/builtins/trunctfsf2.c index de96c1decf63..0261b1e90f5d 100644 --- a/lib/builtins/trunctfsf2.c +++ b/lib/builtins/trunctfsf2.c @@ -1,9 +1,8 @@ //===-- lib/trunctfsf2.c - quad -> single conversion --------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,8 +14,6 @@ #define DST_SINGLE #include "fp_trunc_impl.inc" -COMPILER_RT_ABI float __trunctfsf2(long double a) { - return __truncXfYf2__(a); -} +COMPILER_RT_ABI float __trunctfsf2(long double a) { return __truncXfYf2__(a); } #endif diff --git a/lib/builtins/ucmpdi2.c b/lib/builtins/ucmpdi2.c index 40af23613b1f..0f2edde8358b 100644 --- a/lib/builtins/ucmpdi2.c +++ b/lib/builtins/ucmpdi2.c @@ -1,51 +1,42 @@ -/* ===-- ucmpdi2.c - Implement __ucmpdi2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ucmpdi2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ucmpdi2.c - Implement __ucmpdi2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ucmpdi2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: if (a < b) returns 0 - * if (a == b) returns 1 - * if (a > b) returns 2 - */ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__ucmpdi2(du_int a, du_int b) -{ - udwords x; - x.all = a; - udwords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __ucmpdi2(du_int a, du_int b) { + udwords x; + x.all = a; + udwords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } #ifdef __ARM_EABI__ -/* Returns: if (a < b) returns -1 -* if (a == b) returns 0 -* if (a > b) returns 1 -*/ -COMPILER_RT_ABI si_int -__aeabi_ulcmp(di_int a, di_int b) -{ - return __ucmpdi2(a, b) - 1; +// Returns: if (a < b) returns -1 +// if (a == b) returns 0 +// if (a > b) returns 1 +COMPILER_RT_ABI si_int __aeabi_ulcmp(di_int a, di_int b) { + return __ucmpdi2(a, b) - 1; } #endif - diff --git a/lib/builtins/ucmpti2.c b/lib/builtins/ucmpti2.c index bda8083bb2a1..4eb6655b05af 100644 --- a/lib/builtins/ucmpti2.c +++ b/lib/builtins/ucmpti2.c @@ -1,42 +1,37 @@ -/* ===-- ucmpti2.c - Implement __ucmpti2 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __ucmpti2 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- ucmpti2.c - Implement __ucmpti2 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __ucmpti2 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: if (a < b) returns 0 - * if (a == b) returns 1 - * if (a > b) returns 2 - */ +// Returns: if (a < b) returns 0 +// if (a == b) returns 1 +// if (a > b) returns 2 -COMPILER_RT_ABI si_int -__ucmpti2(tu_int a, tu_int b) -{ - utwords x; - x.all = a; - utwords y; - y.all = b; - if (x.s.high < y.s.high) - return 0; - if (x.s.high > y.s.high) - return 2; - if (x.s.low < y.s.low) - return 0; - if (x.s.low > y.s.low) - return 2; - return 1; +COMPILER_RT_ABI si_int __ucmpti2(tu_int a, tu_int b) { + utwords x; + x.all = a; + utwords y; + y.all = b; + if (x.s.high < y.s.high) + return 0; + if (x.s.high > y.s.high) + return 2; + if (x.s.low < y.s.low) + return 0; + if (x.s.low > y.s.low) + return 2; + return 1; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/udivdi3.c b/lib/builtins/udivdi3.c index dc68e154b109..a23139ec947f 100644 --- a/lib/builtins/udivdi3.c +++ b/lib/builtins/udivdi3.c @@ -1,23 +1,19 @@ -/* ===-- udivdi3.c - Implement __udivdi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivdi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivdi3.c - Implement __udivdi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivdi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI du_int -__udivdi3(du_int a, du_int b) -{ - return __udivmoddi4(a, b, 0); +COMPILER_RT_ABI du_int __udivdi3(du_int a, du_int b) { + return __udivmoddi4(a, b, 0); } diff --git a/lib/builtins/udivmoddi4.c b/lib/builtins/udivmoddi4.c index 0c8b4ff46474..2914cc0fb46d 100644 --- a/lib/builtins/udivmoddi4.c +++ b/lib/builtins/udivmoddi4.c @@ -1,231 +1,189 @@ -/* ===-- udivmoddi4.c - Implement __udivmoddi4 -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivmoddi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivmoddi4.c - Implement __udivmoddi4 -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivmoddi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Effects: if rem != 0, *rem = a % b - * Returns: a / b - */ +// Effects: if rem != 0, *rem = a % b +// Returns: a / b -/* Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide */ +// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide -COMPILER_RT_ABI du_int -__udivmoddi4(du_int a, du_int b, du_int* rem) -{ - const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; - const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; - udwords n; - n.all = a; - udwords d; - d.all = b; - udwords q; - udwords r; - unsigned sr; - /* special cases, X is unknown, K != 0 */ - if (n.s.high == 0) - { - if (d.s.high == 0) - { - /* 0 X - * --- - * 0 X - */ - if (rem) - *rem = n.s.low % d.s.low; - return n.s.low / d.s.low; - } - /* 0 X - * --- - * K X - */ - if (rem) - *rem = n.s.low; - return 0; +COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem) { + const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; + const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; + udwords n; + n.all = a; + udwords d; + d.all = b; + udwords q; + udwords r; + unsigned sr; + // special cases, X is unknown, K != 0 + if (n.s.high == 0) { + if (d.s.high == 0) { + // 0 X + // --- + // 0 X + if (rem) + *rem = n.s.low % d.s.low; + return n.s.low / d.s.low; + } + // 0 X + // --- + // K X + if (rem) + *rem = n.s.low; + return 0; + } + // n.s.high != 0 + if (d.s.low == 0) { + if (d.s.high == 0) { + // K X + // --- + // 0 0 + if (rem) + *rem = n.s.high % d.s.low; + return n.s.high / d.s.low; + } + // d.s.high != 0 + if (n.s.low == 0) { + // K 0 + // --- + // K 0 + if (rem) { + r.s.high = n.s.high % d.s.high; + r.s.low = 0; + *rem = r.all; + } + return n.s.high / d.s.high; + } + // K K + // --- + // K 0 + if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ { + if (rem) { + r.s.low = n.s.low; + r.s.high = n.s.high & (d.s.high - 1); + *rem = r.all; + } + return n.s.high >> __builtin_ctz(d.s.high); } - /* n.s.high != 0 */ - if (d.s.low == 0) - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 0 - */ - if (rem) - *rem = n.s.high % d.s.low; - return n.s.high / d.s.low; - } - /* d.s.high != 0 */ - if (n.s.low == 0) - { - /* K 0 - * --- - * K 0 - */ - if (rem) - { - r.s.high = n.s.high % d.s.high; - r.s.low = 0; - *rem = r.all; - } - return n.s.high / d.s.high; - } - /* K K - * --- - * K 0 - */ - if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - { - r.s.low = n.s.low; - r.s.high = n.s.high & (d.s.high - 1); - *rem = r.all; - } - return n.s.high >> __builtin_ctz(d.s.high); - } - /* K K - * --- - * K 0 - */ - sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); - /* 0 <= sr <= n_uword_bits - 2 or sr large */ - if (sr > n_uword_bits - 2) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_uword_bits - 1 */ - /* q.all = n.all << (n_udword_bits - sr); */ + // K K + // --- + // K 0 + sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); + // 0 <= sr <= n_uword_bits - 2 or sr large + if (sr > n_uword_bits - 2) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_uword_bits - 1 + // q.all = n.all << (n_udword_bits - sr); + q.s.low = 0; + q.s.high = n.s.low << (n_uword_bits - sr); + // r.all = n.all >> sr; + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } else /* d.s.low != 0 */ { + if (d.s.high == 0) { + // K X + // --- + // 0 K + if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ { + if (rem) + *rem = n.s.low & (d.s.low - 1); + if (d.s.low == 1) + return n.all; + sr = __builtin_ctz(d.s.low); + q.s.high = n.s.high >> sr; + q.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + return q.all; + } + // K X + // --- + // 0 K + sr = 1 + n_uword_bits + __builtin_clz(d.s.low) - __builtin_clz(n.s.high); + // 2 <= sr <= n_udword_bits - 1 + // q.all = n.all << (n_udword_bits - sr); + // r.all = n.all >> sr; + if (sr == n_uword_bits) { + q.s.low = 0; + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else if (sr < n_uword_bits) /* 2 <= sr <= n_uword_bits - 1 */ { q.s.low = 0; q.s.high = n.s.low << (n_uword_bits - sr); - /* r.all = n.all >> sr; */ r.s.high = n.s.high >> sr; r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } else /* n_uword_bits + 1 <= sr <= n_udword_bits - 1 */ { + q.s.low = n.s.low << (n_udword_bits - sr); + q.s.high = (n.s.high << (n_udword_bits - sr)) | + (n.s.low >> (sr - n_uword_bits)); + r.s.high = 0; + r.s.low = n.s.high >> (sr - n_uword_bits); + } + } else { + // K X + // --- + // K K + sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); + // 0 <= sr <= n_uword_bits - 1 or sr large + if (sr > n_uword_bits - 1) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_uword_bits + // q.all = n.all << (n_udword_bits - sr); + q.s.low = 0; + if (sr == n_uword_bits) { + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else { + q.s.high = n.s.low << (n_uword_bits - sr); + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); + } } - else /* d.s.low != 0 */ - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 K - */ - if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - *rem = n.s.low & (d.s.low - 1); - if (d.s.low == 1) - return n.all; - sr = __builtin_ctz(d.s.low); - q.s.high = n.s.high >> sr; - q.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); - return q.all; - } - /* K X - * --- - * 0 K - */ - sr = 1 + n_uword_bits + __builtin_clz(d.s.low) - __builtin_clz(n.s.high); - /* 2 <= sr <= n_udword_bits - 1 - * q.all = n.all << (n_udword_bits - sr); - * r.all = n.all >> sr; - */ - if (sr == n_uword_bits) - { - q.s.low = 0; - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else if (sr < n_uword_bits) // 2 <= sr <= n_uword_bits - 1 - { - q.s.low = 0; - q.s.high = n.s.low << (n_uword_bits - sr); - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); - } - else // n_uword_bits + 1 <= sr <= n_udword_bits - 1 - { - q.s.low = n.s.low << (n_udword_bits - sr); - q.s.high = (n.s.high << (n_udword_bits - sr)) | - (n.s.low >> (sr - n_uword_bits)); - r.s.high = 0; - r.s.low = n.s.high >> (sr - n_uword_bits); - } - } - else - { - /* K X - * --- - * K K - */ - sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high); - /* 0 <= sr <= n_uword_bits - 1 or sr large */ - if (sr > n_uword_bits - 1) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_uword_bits */ - /* q.all = n.all << (n_udword_bits - sr); */ - q.s.low = 0; - if (sr == n_uword_bits) - { - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else - { - q.s.high = n.s.low << (n_uword_bits - sr); - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr); - } - } - } - /* Not a special case - * q and r are initialized with: - * q.all = n.all << (n_udword_bits - sr); - * r.all = n.all >> sr; - * 1 <= sr <= n_udword_bits - 1 - */ - su_int carry = 0; - for (; sr > 0; --sr) - { - /* r:q = ((r:q) << 1) | carry */ - r.s.high = (r.s.high << 1) | (r.s.low >> (n_uword_bits - 1)); - r.s.low = (r.s.low << 1) | (q.s.high >> (n_uword_bits - 1)); - q.s.high = (q.s.high << 1) | (q.s.low >> (n_uword_bits - 1)); - q.s.low = (q.s.low << 1) | carry; - /* carry = 0; - * if (r.all >= d.all) - * { - * r.all -= d.all; - * carry = 1; - * } - */ - const di_int s = (di_int)(d.all - r.all - 1) >> (n_udword_bits - 1); - carry = s & 1; - r.all -= d.all & s; - } - q.all = (q.all << 1) | carry; - if (rem) - *rem = r.all; - return q.all; + } + // Not a special case + // q and r are initialized with: + // q.all = n.all << (n_udword_bits - sr); + // r.all = n.all >> sr; + // 1 <= sr <= n_udword_bits - 1 + su_int carry = 0; + for (; sr > 0; --sr) { + // r:q = ((r:q) << 1) | carry + r.s.high = (r.s.high << 1) | (r.s.low >> (n_uword_bits - 1)); + r.s.low = (r.s.low << 1) | (q.s.high >> (n_uword_bits - 1)); + q.s.high = (q.s.high << 1) | (q.s.low >> (n_uword_bits - 1)); + q.s.low = (q.s.low << 1) | carry; + // carry = 0; + // if (r.all >= d.all) + // { + // r.all -= d.all; + // carry = 1; + // } + const di_int s = (di_int)(d.all - r.all - 1) >> (n_udword_bits - 1); + carry = s & 1; + r.all -= d.all & s; + } + q.all = (q.all << 1) | carry; + if (rem) + *rem = r.all; + return q.all; } diff --git a/lib/builtins/udivmodsi4.c b/lib/builtins/udivmodsi4.c index 789c4b5061e4..753ad6dd96db 100644 --- a/lib/builtins/udivmodsi4.c +++ b/lib/builtins/udivmodsi4.c @@ -1,27 +1,21 @@ -/*===-- udivmodsi4.c - Implement __udivmodsi4 ------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivmodsi4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivmodsi4.c - Implement __udivmodsi4 -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivmodsi4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b, *rem = a % b */ +// Returns: a / b, *rem = a % b -COMPILER_RT_ABI su_int -__udivmodsi4(su_int a, su_int b, su_int* rem) -{ - si_int d = __udivsi3(a,b); - *rem = a - (d*b); +COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int *rem) { + si_int d = __udivsi3(a, b); + *rem = a - (d * b); return d; } - - diff --git a/lib/builtins/udivmodti4.c b/lib/builtins/udivmodti4.c index 803168849c6c..dd14a8b579ca 100644 --- a/lib/builtins/udivmodti4.c +++ b/lib/builtins/udivmodti4.c @@ -1,238 +1,195 @@ -/* ===-- udivmodti4.c - Implement __udivmodti4 -----------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivmodti4 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivmodti4.c - Implement __udivmodti4 -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivmodti4 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Effects: if rem != 0, *rem = a % b - * Returns: a / b - */ +// Effects: if rem != 0, *rem = a % b +// Returns: a / b -/* Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide */ +// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide -COMPILER_RT_ABI tu_int -__udivmodti4(tu_int a, tu_int b, tu_int* rem) -{ - const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; - const unsigned n_utword_bits = sizeof(tu_int) * CHAR_BIT; - utwords n; - n.all = a; - utwords d; - d.all = b; - utwords q; - utwords r; - unsigned sr; - /* special cases, X is unknown, K != 0 */ - if (n.s.high == 0) - { - if (d.s.high == 0) - { - /* 0 X - * --- - * 0 X - */ - if (rem) - *rem = n.s.low % d.s.low; - return n.s.low / d.s.low; - } - /* 0 X - * --- - * K X - */ - if (rem) - *rem = n.s.low; - return 0; +COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem) { + const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT; + const unsigned n_utword_bits = sizeof(tu_int) * CHAR_BIT; + utwords n; + n.all = a; + utwords d; + d.all = b; + utwords q; + utwords r; + unsigned sr; + // special cases, X is unknown, K != 0 + if (n.s.high == 0) { + if (d.s.high == 0) { + // 0 X + // --- + // 0 X + if (rem) + *rem = n.s.low % d.s.low; + return n.s.low / d.s.low; + } + // 0 X + // --- + // K X + if (rem) + *rem = n.s.low; + return 0; + } + // n.s.high != 0 + if (d.s.low == 0) { + if (d.s.high == 0) { + // K X + // --- + // 0 0 + if (rem) + *rem = n.s.high % d.s.low; + return n.s.high / d.s.low; + } + // d.s.high != 0 + if (n.s.low == 0) { + // K 0 + // --- + // K 0 + if (rem) { + r.s.high = n.s.high % d.s.high; + r.s.low = 0; + *rem = r.all; + } + return n.s.high / d.s.high; + } + // K K + // --- + // K 0 + if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ { + if (rem) { + r.s.low = n.s.low; + r.s.high = n.s.high & (d.s.high - 1); + *rem = r.all; + } + return n.s.high >> __builtin_ctzll(d.s.high); } - /* n.s.high != 0 */ - if (d.s.low == 0) - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 0 - */ - if (rem) - *rem = n.s.high % d.s.low; - return n.s.high / d.s.low; - } - /* d.s.high != 0 */ - if (n.s.low == 0) - { - /* K 0 - * --- - * K 0 - */ - if (rem) - { - r.s.high = n.s.high % d.s.high; - r.s.low = 0; - *rem = r.all; - } - return n.s.high / d.s.high; - } - /* K K - * --- - * K 0 - */ - if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - { - r.s.low = n.s.low; - r.s.high = n.s.high & (d.s.high - 1); - *rem = r.all; - } - return n.s.high >> __builtin_ctzll(d.s.high); - } - /* K K - * --- - * K 0 - */ - sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); - /* 0 <= sr <= n_udword_bits - 2 or sr large */ - if (sr > n_udword_bits - 2) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_udword_bits - 1 */ - /* q.all = n.all << (n_utword_bits - sr); */ + // K K + // --- + // K 0 + sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); + // 0 <= sr <= n_udword_bits - 2 or sr large + if (sr > n_udword_bits - 2) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_udword_bits - 1 + // q.all = n.all << (n_utword_bits - sr); + q.s.low = 0; + q.s.high = n.s.low << (n_udword_bits - sr); + // r.all = n.all >> sr; + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + } else /* d.s.low != 0 */ { + if (d.s.high == 0) { + // K X + // --- + // 0 K + if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ { + if (rem) + *rem = n.s.low & (d.s.low - 1); + if (d.s.low == 1) + return n.all; + sr = __builtin_ctzll(d.s.low); + q.s.high = n.s.high >> sr; + q.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + return q.all; + } + // K X + // --- + // 0 K + sr = 1 + n_udword_bits + __builtin_clzll(d.s.low) - + __builtin_clzll(n.s.high); + // 2 <= sr <= n_utword_bits - 1 + // q.all = n.all << (n_utword_bits - sr); + // r.all = n.all >> sr; + if (sr == n_udword_bits) { + q.s.low = 0; + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else if (sr < n_udword_bits) /* 2 <= sr <= n_udword_bits - 1 */ { q.s.low = 0; q.s.high = n.s.low << (n_udword_bits - sr); - /* r.all = n.all >> sr; */ r.s.high = n.s.high >> sr; r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + } else /* n_udword_bits + 1 <= sr <= n_utword_bits - 1 */ { + q.s.low = n.s.low << (n_utword_bits - sr); + q.s.high = (n.s.high << (n_utword_bits - sr)) | + (n.s.low >> (sr - n_udword_bits)); + r.s.high = 0; + r.s.low = n.s.high >> (sr - n_udword_bits); + } + } else { + // K X + // --- + // K K + sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); + // 0 <= sr <= n_udword_bits - 1 or sr large + if (sr > n_udword_bits - 1) { + if (rem) + *rem = n.all; + return 0; + } + ++sr; + // 1 <= sr <= n_udword_bits + // q.all = n.all << (n_utword_bits - sr); + // r.all = n.all >> sr; + q.s.low = 0; + if (sr == n_udword_bits) { + q.s.high = n.s.low; + r.s.high = 0; + r.s.low = n.s.high; + } else { + r.s.high = n.s.high >> sr; + r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); + q.s.high = n.s.low << (n_udword_bits - sr); + } } - else /* d.s.low != 0 */ - { - if (d.s.high == 0) - { - /* K X - * --- - * 0 K - */ - if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ - { - if (rem) - *rem = n.s.low & (d.s.low - 1); - if (d.s.low == 1) - return n.all; - sr = __builtin_ctzll(d.s.low); - q.s.high = n.s.high >> sr; - q.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); - return q.all; - } - /* K X - * --- - * 0 K - */ - sr = 1 + n_udword_bits + __builtin_clzll(d.s.low) - - __builtin_clzll(n.s.high); - /* 2 <= sr <= n_utword_bits - 1 - * q.all = n.all << (n_utword_bits - sr); - * r.all = n.all >> sr; - */ - if (sr == n_udword_bits) - { - q.s.low = 0; - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else if (sr < n_udword_bits) // 2 <= sr <= n_udword_bits - 1 - { - q.s.low = 0; - q.s.high = n.s.low << (n_udword_bits - sr); - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); - } - else // n_udword_bits + 1 <= sr <= n_utword_bits - 1 - { - q.s.low = n.s.low << (n_utword_bits - sr); - q.s.high = (n.s.high << (n_utword_bits - sr)) | - (n.s.low >> (sr - n_udword_bits)); - r.s.high = 0; - r.s.low = n.s.high >> (sr - n_udword_bits); - } - } - else - { - /* K X - * --- - * K K - */ - sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high); - /*0 <= sr <= n_udword_bits - 1 or sr large */ - if (sr > n_udword_bits - 1) - { - if (rem) - *rem = n.all; - return 0; - } - ++sr; - /* 1 <= sr <= n_udword_bits - * q.all = n.all << (n_utword_bits - sr); - * r.all = n.all >> sr; - */ - q.s.low = 0; - if (sr == n_udword_bits) - { - q.s.high = n.s.low; - r.s.high = 0; - r.s.low = n.s.high; - } - else - { - r.s.high = n.s.high >> sr; - r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr); - q.s.high = n.s.low << (n_udword_bits - sr); - } - } - } - /* Not a special case - * q and r are initialized with: - * q.all = n.all << (n_utword_bits - sr); - * r.all = n.all >> sr; - * 1 <= sr <= n_utword_bits - 1 - */ - su_int carry = 0; - for (; sr > 0; --sr) - { - /* r:q = ((r:q) << 1) | carry */ - r.s.high = (r.s.high << 1) | (r.s.low >> (n_udword_bits - 1)); - r.s.low = (r.s.low << 1) | (q.s.high >> (n_udword_bits - 1)); - q.s.high = (q.s.high << 1) | (q.s.low >> (n_udword_bits - 1)); - q.s.low = (q.s.low << 1) | carry; - /* carry = 0; - * if (r.all >= d.all) - * { - * r.all -= d.all; - * carry = 1; - * } - */ - const ti_int s = (ti_int)(d.all - r.all - 1) >> (n_utword_bits - 1); - carry = s & 1; - r.all -= d.all & s; - } - q.all = (q.all << 1) | carry; - if (rem) - *rem = r.all; - return q.all; + } + // Not a special case + // q and r are initialized with: + // q.all = n.all << (n_utword_bits - sr); + // r.all = n.all >> sr; + // 1 <= sr <= n_utword_bits - 1 + su_int carry = 0; + for (; sr > 0; --sr) { + // r:q = ((r:q) << 1) | carry + r.s.high = (r.s.high << 1) | (r.s.low >> (n_udword_bits - 1)); + r.s.low = (r.s.low << 1) | (q.s.high >> (n_udword_bits - 1)); + q.s.high = (q.s.high << 1) | (q.s.low >> (n_udword_bits - 1)); + q.s.low = (q.s.low << 1) | carry; + // carry = 0; + // if (r.all >= d.all) + // { + // r.all -= d.all; + // carry = 1; + // } + const ti_int s = (ti_int)(d.all - r.all - 1) >> (n_utword_bits - 1); + carry = s & 1; + r.all -= d.all & s; + } + q.all = (q.all << 1) | carry; + if (rem) + *rem = r.all; + return q.all; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/udivsi3.c b/lib/builtins/udivsi3.c index bb720f8c382b..18cc96c1b2e0 100644 --- a/lib/builtins/udivsi3.c +++ b/lib/builtins/udivsi3.c @@ -1,68 +1,62 @@ -/* ===-- udivsi3.c - Implement __udivsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivsi3.c - Implement __udivsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a / b */ +// Returns: a / b -/* Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide */ +// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide -/* This function should not call __divsi3! */ -COMPILER_RT_ABI su_int -__udivsi3(su_int n, su_int d) -{ - const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; - su_int q; - su_int r; - unsigned sr; - /* special cases */ - if (d == 0) - return 0; /* ?! */ - if (n == 0) - return 0; - sr = __builtin_clz(d) - __builtin_clz(n); - /* 0 <= sr <= n_uword_bits - 1 or sr large */ - if (sr > n_uword_bits - 1) /* d > r */ - return 0; - if (sr == n_uword_bits - 1) /* d == 1 */ - return n; - ++sr; - /* 1 <= sr <= n_uword_bits - 1 */ - /* Not a special case */ - q = n << (n_uword_bits - sr); - r = n >> sr; - su_int carry = 0; - for (; sr > 0; --sr) - { - /* r:q = ((r:q) << 1) | carry */ - r = (r << 1) | (q >> (n_uword_bits - 1)); - q = (q << 1) | carry; - /* carry = 0; - * if (r.all >= d.all) - * { - * r.all -= d.all; - * carry = 1; - * } - */ - const si_int s = (si_int)(d - r - 1) >> (n_uword_bits - 1); - carry = s & 1; - r -= d & s; - } +// This function should not call __divsi3! +COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d) { + const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT; + su_int q; + su_int r; + unsigned sr; + // special cases + if (d == 0) + return 0; // ?! + if (n == 0) + return 0; + sr = __builtin_clz(d) - __builtin_clz(n); + // 0 <= sr <= n_uword_bits - 1 or sr large + if (sr > n_uword_bits - 1) // d > r + return 0; + if (sr == n_uword_bits - 1) // d == 1 + return n; + ++sr; + // 1 <= sr <= n_uword_bits - 1 + // Not a special case + q = n << (n_uword_bits - sr); + r = n >> sr; + su_int carry = 0; + for (; sr > 0; --sr) { + // r:q = ((r:q) << 1) | carry + r = (r << 1) | (q >> (n_uword_bits - 1)); q = (q << 1) | carry; - return q; + // carry = 0; + // if (r.all >= d.all) + // { + // r.all -= d.all; + // carry = 1; + // } + const si_int s = (si_int)(d - r - 1) >> (n_uword_bits - 1); + carry = s & 1; + r -= d & s; + } + q = (q << 1) | carry; + return q; } #if defined(__ARM_EABI__) -AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) COMPILER_RT_ALIAS(__udivsi3); +COMPILER_RT_ALIAS(__udivsi3, __aeabi_uidiv) #endif diff --git a/lib/builtins/udivti3.c b/lib/builtins/udivti3.c index ec94673e25b0..4c82040b88cb 100644 --- a/lib/builtins/udivti3.c +++ b/lib/builtins/udivti3.c @@ -1,27 +1,23 @@ -/* ===-- udivti3.c - Implement __udivti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __udivti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- udivti3.c - Implement __udivti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __udivti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a / b */ +// Returns: a / b -COMPILER_RT_ABI tu_int -__udivti3(tu_int a, tu_int b) -{ - return __udivmodti4(a, b, 0); +COMPILER_RT_ABI tu_int __udivti3(tu_int a, tu_int b) { + return __udivmodti4(a, b, 0); } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/umoddi3.c b/lib/builtins/umoddi3.c index d513f080a1e9..965cf8fc01bd 100644 --- a/lib/builtins/umoddi3.c +++ b/lib/builtins/umoddi3.c @@ -1,25 +1,21 @@ -/* ===-- umoddi3.c - Implement __umoddi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __umoddi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- umoddi3.c - Implement __umoddi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __umoddi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI du_int -__umoddi3(du_int a, du_int b) -{ - du_int r; - __udivmoddi4(a, b, &r); - return r; +COMPILER_RT_ABI du_int __umoddi3(du_int a, du_int b) { + du_int r; + __udivmoddi4(a, b, &r); + return r; } diff --git a/lib/builtins/umodsi3.c b/lib/builtins/umodsi3.c index d5fda4a6af1f..ce9abcd94ef7 100644 --- a/lib/builtins/umodsi3.c +++ b/lib/builtins/umodsi3.c @@ -1,23 +1,19 @@ -/* ===-- umodsi3.c - Implement __umodsi3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __umodsi3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- umodsi3.c - Implement __umodsi3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __umodsi3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI su_int -__umodsi3(su_int a, su_int b) -{ - return a - __udivsi3(a, b) * b; +COMPILER_RT_ABI su_int __umodsi3(su_int a, su_int b) { + return a - __udivsi3(a, b) * b; } diff --git a/lib/builtins/umodti3.c b/lib/builtins/umodti3.c index 6d1ca7a8cf6f..8cc5cb6b88f7 100644 --- a/lib/builtins/umodti3.c +++ b/lib/builtins/umodti3.c @@ -1,29 +1,25 @@ -/* ===-- umodti3.c - Implement __umodti3 -----------------------------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file implements __umodti3 for the compiler_rt library. - * - * ===----------------------------------------------------------------------=== - */ +//===-- umodti3.c - Implement __umodti3 -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements __umodti3 for the compiler_rt library. +// +//===----------------------------------------------------------------------===// #include "int_lib.h" #ifdef CRT_HAS_128BIT -/* Returns: a % b */ +// Returns: a % b -COMPILER_RT_ABI tu_int -__umodti3(tu_int a, tu_int b) -{ - tu_int r; - __udivmodti4(a, b, &r); - return r; +COMPILER_RT_ABI tu_int __umodti3(tu_int a, tu_int b) { + tu_int r; + __udivmodti4(a, b, &r); + return r; } -#endif /* CRT_HAS_128BIT */ +#endif // CRT_HAS_128BIT diff --git a/lib/builtins/unwind-ehabi-helpers.h b/lib/builtins/unwind-ehabi-helpers.h index ccb0765975a9..1b48cdb75e1b 100644 --- a/lib/builtins/unwind-ehabi-helpers.h +++ b/lib/builtins/unwind-ehabi-helpers.h @@ -1,43 +1,40 @@ -/* ===-- arm-ehabi-helpers.h - Supplementary ARM EHABI declarations --------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===--------------------------------------------------------------------=== */ +//===-- arm-ehabi-helpers.h - Supplementary ARM EHABI declarations --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===--------------------------------------------------------------------===// #ifndef UNWIND_EHABI_HELPERS_H #define UNWIND_EHABI_HELPERS_H #include <stdint.h> -/* NOTE: see reasoning for this inclusion below */ +// NOTE: see reasoning for this inclusion below #include <unwind.h> #if !defined(__ARM_EABI_UNWINDER__) -/* - * NOTE: _URC_OK, _URC_FAILURE must be present as preprocessor tokens. This - * allows for a substitution of a constant which can be cast into the - * appropriate enumerated type. This header is expected to always be included - * AFTER unwind.h (which is why it is forcefully included above). This ensures - * that we do not overwrite the token for the enumeration. Subsequent uses of - * the token would be clean to rewrite with constant values. - * - * The typedef redeclaration should be safe. Due to the protection granted to - * us by the `__ARM_EABI_UNWINDER__` above, we are guaranteed that we are in a - * header not vended by gcc. The HP unwinder (being an itanium unwinder) does - * not support EHABI, and the GNU unwinder, derived from the HP unwinder, also - * does not support EHABI as of the introduction of this header. As such, we - * are fairly certain that we are in the LLVM case. Here, _Unwind_State is a - * typedef, and so we can get away with a redeclaration. - * - * Guarded redefinitions of the needed unwind state prevent the redefinition of - * those states. - */ - -#define _URC_OK 0 -#define _URC_FAILURE 9 +// NOTE: _URC_OK, _URC_FAILURE must be present as preprocessor tokens. This +// allows for a substitution of a constant which can be cast into the +// appropriate enumerated type. This header is expected to always be included +// AFTER unwind.h (which is why it is forcefully included above). This ensures +// that we do not overwrite the token for the enumeration. Subsequent uses of +// the token would be clean to rewrite with constant values. +// +// The typedef redeclaration should be safe. Due to the protection granted to +// us by the `__ARM_EABI_UNWINDER__` above, we are guaranteed that we are in a +// header not vended by gcc. The HP unwinder (being an itanium unwinder) does +// not support EHABI, and the GNU unwinder, derived from the HP unwinder, also +// does not support EHABI as of the introduction of this header. As such, we +// are fairly certain that we are in the LLVM case. Here, _Unwind_State is a +// typedef, and so we can get away with a redeclaration. +// +// Guarded redefinitions of the needed unwind state prevent the redefinition of +// those states. + +#define _URC_OK 0 +#define _URC_FAILURE 9 typedef uint32_t _Unwind_State; @@ -52,4 +49,3 @@ typedef uint32_t _Unwind_State; #endif #endif - diff --git a/lib/builtins/x86_64/chkstk.S b/lib/builtins/x86_64/chkstk.S index 4149ac63d9d0..ad7953a116ac 100644 --- a/lib/builtins/x86_64/chkstk.S +++ b/lib/builtins/x86_64/chkstk.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/x86_64/chkstk2.S b/lib/builtins/x86_64/chkstk2.S index ac1eb920e0e8..33d10d5b63be 100644 --- a/lib/builtins/x86_64/chkstk2.S +++ b/lib/builtins/x86_64/chkstk2.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" diff --git a/lib/builtins/x86_64/floatdidf.c b/lib/builtins/x86_64/floatdidf.c index dead0ed42c65..f83f53a38e9e 100644 --- a/lib/builtins/x86_64/floatdidf.c +++ b/lib/builtins/x86_64/floatdidf.c @@ -1,16 +1,13 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* double __floatdidf(di_int a); */ +// double __floatdidf(di_int a); #if defined(__x86_64__) || defined(_M_X64) #include "../int_lib.h" -double __floatdidf(int64_t a) -{ - return (double)a; -} +double __floatdidf(int64_t a) { return (double)a; } -#endif /* __x86_64__ */ +#endif // __x86_64__ diff --git a/lib/builtins/x86_64/floatdisf.c b/lib/builtins/x86_64/floatdisf.c index 99d5621c6327..06c118cfa18d 100644 --- a/lib/builtins/x86_64/floatdisf.c +++ b/lib/builtins/x86_64/floatdisf.c @@ -1,14 +1,11 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #if defined(__x86_64__) || defined(_M_X64) #include "../int_lib.h" -float __floatdisf(int64_t a) -{ - return (float)a; -} +float __floatdisf(int64_t a) { return (float)a; } -#endif /* __x86_64__ */ +#endif // __x86_64__ diff --git a/lib/builtins/x86_64/floatdixf.c b/lib/builtins/x86_64/floatdixf.c index c01193a82b5e..cf8450ce6f42 100644 --- a/lib/builtins/x86_64/floatdixf.c +++ b/lib/builtins/x86_64/floatdixf.c @@ -1,16 +1,13 @@ -/* This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. - */ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/* long double __floatdixf(di_int a); */ +// long double __floatdixf(di_int a); #ifdef __x86_64__ #include "../int_lib.h" -long double __floatdixf(int64_t a) -{ - return (long double)a; -} +long double __floatdixf(int64_t a) { return (long double)a; } -#endif /* __i386__ */ +#endif // __i386__ diff --git a/lib/builtins/x86_64/floatundidf.S b/lib/builtins/x86_64/floatundidf.S index 094a68dc3cd4..7f6ef3bbb6bf 100644 --- a/lib/builtins/x86_64/floatundidf.S +++ b/lib/builtins/x86_64/floatundidf.S @@ -1,9 +1,8 @@ //===-- floatundidf.S - Implement __floatundidf for x86_64 ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/builtins/x86_64/floatundisf.S b/lib/builtins/x86_64/floatundisf.S index 7c9f75e188eb..246bdff5a4c3 100644 --- a/lib/builtins/x86_64/floatundisf.S +++ b/lib/builtins/x86_64/floatundisf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -23,7 +24,7 @@ DEFINE_COMPILERRT_FUNCTION(__floatundisf) js 1f cvtsi2ssq %rdi, %xmm0 ret - + 1: andq %rdi, %rsi shrq %rdi orq %rsi, %rdi diff --git a/lib/builtins/x86_64/floatundixf.S b/lib/builtins/x86_64/floatundixf.S index 28a096b71373..9e3bcedcb7e4 100644 --- a/lib/builtins/x86_64/floatundixf.S +++ b/lib/builtins/x86_64/floatundixf.S @@ -1,5 +1,6 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "../assembly.h" @@ -57,7 +58,7 @@ DEFINE_COMPILERRT_FUNCTION(__floatundixf) orq REL_ADDR(twop52), %rsi // 2^52 + lo (as a double) movq %rdi, -8(%rsp) movq %rsi, -16(%rsp) - fldl REL_ADDR(twop84_plus_twop52_neg) + fldl REL_ADDR(twop84_plus_twop52_neg) faddl -8(%rsp) // hi - 2^52 (as double extended, no rounding occurs) faddl -16(%rsp) // hi + lo (as double extended) ret diff --git a/lib/cfi/cfi.cc b/lib/cfi/cfi.cpp index b0a943759678..fd48f71643b6 100644 --- a/lib/cfi/cfi.cc +++ b/lib/cfi/cfi.cpp @@ -1,9 +1,8 @@ -//===-------- cfi.cc ------------------------------------------------------===// +//===-------- cfi.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -187,7 +186,7 @@ void ShadowBuilder::Install() { GetShadowSize(), MAP_FIXED); CHECK(res != MAP_FAILED); #else - void *res = MmapFixedOrDie(shadow_, GetShadowSize()); + void *res = MmapFixedOrDie(shadow_, GetShadowSize(), "cfi shadow"); CHECK(res != MAP_FAILED); ::memcpy(&shadow_, &main_shadow, GetShadowSize()); #endif @@ -373,7 +372,7 @@ void InitializeFlags() { FlagParser cfi_parser; RegisterCommonFlags(&cfi_parser); - cfi_parser.ParseString(GetEnv("CFI_OPTIONS")); + cfi_parser.ParseStringFromEnv("CFI_OPTIONS"); #ifdef CFI_ENABLE_DIAG FlagParser ubsan_parser; @@ -382,7 +381,7 @@ void InitializeFlags() { const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); ubsan_parser.ParseString(ubsan_default_options); - ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); #endif InitializeCommonFlags(); diff --git a/lib/crt/crtbegin.c b/lib/crt/crtbegin.c new file mode 100644 index 000000000000..2450ce54e31b --- /dev/null +++ b/lib/crt/crtbegin.c @@ -0,0 +1,97 @@ +//===-- crtbegin.c - Start of constructors and destructors ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <stddef.h> + +__attribute__((visibility("hidden"))) void *__dso_handle = &__dso_handle; + +__extension__ static void *__EH_FRAME_LIST__[] + __attribute__((section(".eh_frame"), aligned(sizeof(void *)))) = {}; + +extern void __register_frame_info(const void *, void *) __attribute__((weak)); +extern void *__deregister_frame_info(const void *) __attribute__((weak)); + +#ifndef CRT_HAS_INITFINI_ARRAY +typedef void (*fp)(void); + +static fp __CTOR_LIST__[] + __attribute__((section(".ctors"), aligned(sizeof(fp)))) = {(fp)-1}; +extern fp __CTOR_LIST_END__[]; +#endif + +extern void __cxa_finalize(void *) __attribute__((weak)); + +static void __attribute__((used)) __do_init() { + static _Bool __initialized; + if (__builtin_expect(__initialized, 0)) + return; + __initialized = 1; + + static struct { void *p[8]; } __object; + if (__register_frame_info) + __register_frame_info(__EH_FRAME_LIST__, &__object); + +#ifndef CRT_HAS_INITFINI_ARRAY + const size_t n = __CTOR_LIST_END__ - __CTOR_LIST__ - 1; + for (size_t i = n; i >= 1; i--) __CTOR_LIST__[i](); +#endif +} + +#ifdef CRT_HAS_INITFINI_ARRAY +__attribute__((section(".init_array"), + used)) static void (*__init)(void) = __do_init; +#else // CRT_HAS_INITFINI_ARRAY +#if defined(__i386__) || defined(__x86_64__) +__asm__(".pushsection .init,\"ax\",@progbits\n\t" + "call " __USER_LABEL_PREFIX__ "__do_init\n\t" + ".popsection"); +#elif defined(__arm__) +__asm__(".pushsection .init,\"ax\",%progbits\n\t" + "bl " __USER_LABEL_PREFIX__ "__do_init\n\t" + ".popsection"); +#endif // CRT_HAS_INITFINI_ARRAY +#endif + +#ifndef CRT_HAS_INITFINI_ARRAY +static fp __DTOR_LIST__[] + __attribute__((section(".dtors"), aligned(sizeof(fp)))) = {(fp)-1}; +extern fp __DTOR_LIST_END__[]; +#endif + +static void __attribute__((used)) __do_fini() { + static _Bool __finalized; + if (__builtin_expect(__finalized, 0)) + return; + __finalized = 1; + + if (__cxa_finalize) + __cxa_finalize(__dso_handle); + +#ifndef CRT_HAS_INITFINI_ARRAY + if (__deregister_frame_info) + __deregister_frame_info(__EH_FRAME_LIST__); + + const size_t n = __DTOR_LIST_END__ - __DTOR_LIST__ - 1; + for (size_t i = 1; i <= n; i++) __DTOR_LIST__[i](); +#endif +} + +#ifdef CRT_HAS_INITFINI_ARRAY +__attribute__((section(".fini_array"), + used)) static void (*__fini)(void) = __do_fini; +#else // CRT_HAS_INITFINI_ARRAY +#if defined(__i386__) || defined(__x86_64__) +__asm__(".pushsection .fini,\"ax\",@progbits\n\t" + "call " __USER_LABEL_PREFIX__ "__do_fini\n\t" + ".popsection"); +#elif defined(__arm__) +__asm__(".pushsection .fini,\"ax\",%progbits\n\t" + "bl " __USER_LABEL_PREFIX__ "__do_fini\n\t" + ".popsection"); +#endif +#endif // CRT_HAS_INIT_FINI_ARRAY diff --git a/lib/crt/crtend.c b/lib/crt/crtend.c new file mode 100644 index 000000000000..ebcc60b89a10 --- /dev/null +++ b/lib/crt/crtend.c @@ -0,0 +1,22 @@ +//===-- crtend.c - End of constructors and destructors --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <stdint.h> + +// Put 4-byte zero which is the length field in FDE at the end as a terminator. +const int32_t __EH_FRAME_LIST_END__[] + __attribute__((section(".eh_frame"), aligned(sizeof(int32_t)), + visibility("hidden"), used)) = {0}; + +#ifndef CRT_HAS_INITFINI_ARRAY +typedef void (*fp)(void); +fp __CTOR_LIST_END__[] + __attribute__((section(".ctors"), visibility("hidden"), used)) = {0}; +fp __DTOR_LIST_END__[] + __attribute__((section(".dtors"), visibility("hidden"), used)) = {0}; +#endif diff --git a/lib/dfsan/dfsan.cc b/lib/dfsan/dfsan.cc index 585bdceac764..f4ba1148f782 100644 --- a/lib/dfsan/dfsan.cc +++ b/lib/dfsan/dfsan.cc @@ -1,9 +1,8 @@ //===-- dfsan.cc ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -163,6 +162,8 @@ static void dfsan_check_label(dfsan_label label) { // this function (the instrumentation pass inlines the equality test). extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label __dfsan_union(dfsan_label l1, dfsan_label l2) { + if (flags().fast16labels) + return l1 | l2; DCHECK_NE(l1, l2); if (l1 == 0) @@ -382,7 +383,7 @@ static void InitializeFlags() { FlagParser parser; RegisterCommonFlags(&parser); RegisterDfsanFlags(&parser, &flags()); - parser.ParseString(GetEnv("DFSAN_OPTIONS")); + parser.ParseStringFromEnv("DFSAN_OPTIONS"); InitializeCommonFlags(); if (Verbosity()) ReportUnrecognizedFlags(); if (common_flags()->help) parser.PrintFlagDescriptions(); @@ -420,6 +421,12 @@ static void dfsan_fini() { } } +extern "C" void dfsan_flush() { + UnmapOrDie((void*)ShadowAddr(), UnusedAddr() - ShadowAddr()); + if (!MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr())) + Die(); +} + static void dfsan_init(int argc, char **argv, char **envp) { InitializeFlags(); diff --git a/lib/dfsan/dfsan.h b/lib/dfsan/dfsan.h index 33145deef12e..d662391216e4 100644 --- a/lib/dfsan/dfsan.h +++ b/lib/dfsan/dfsan.h @@ -1,9 +1,8 @@ //===-- dfsan.h -------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan_custom.cc b/lib/dfsan/dfsan_custom.cc index 022aa9a9ab8e..dc7b81da4566 100644 --- a/lib/dfsan/dfsan_custom.cc +++ b/lib/dfsan/dfsan_custom.cc @@ -1,9 +1,8 @@ //===-- dfsan.cc ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan_flags.inc b/lib/dfsan/dfsan_flags.inc index 24fbfcb9e46f..29db73b98278 100644 --- a/lib/dfsan/dfsan_flags.inc +++ b/lib/dfsan/dfsan_flags.inc @@ -1,9 +1,8 @@ //===-- dfsan_flags.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,3 +29,7 @@ DFSAN_FLAG( DFSAN_FLAG(const char *, dump_labels_at_exit, "", "The path of the file where " "to dump the labels when the " "program terminates.") +DFSAN_FLAG(bool, fast16labels, false, + "Enables experimental mode where DFSan supports only 16 power-of-2 labels " + "(1, 2, 4, 8, ... 32768) and the label union is computed as a bit-wise OR." +) diff --git a/lib/dfsan/dfsan_interceptors.cc b/lib/dfsan/dfsan_interceptors.cc index 5ecbb43e7c46..f4b4babc65c2 100644 --- a/lib/dfsan/dfsan_interceptors.cc +++ b/lib/dfsan/dfsan_interceptors.cc @@ -1,9 +1,8 @@ //===-- dfsan_interceptors.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/dfsan_platform.h b/lib/dfsan/dfsan_platform.h index 98284bafd4ff..4ff68b9d40b6 100644 --- a/lib/dfsan/dfsan_platform.h +++ b/lib/dfsan/dfsan_platform.h @@ -1,9 +1,8 @@ //===-- dfsan_platform.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/dfsan/done_abilist.txt b/lib/dfsan/done_abilist.txt index 9765626ad3ff..f8c88dd3201b 100644 --- a/lib/dfsan/done_abilist.txt +++ b/lib/dfsan/done_abilist.txt @@ -26,6 +26,8 @@ fun:dfsan_has_label_with_desc=uninstrumented fun:dfsan_has_label_with_desc=discard fun:dfsan_set_write_callback=uninstrumented fun:dfsan_set_write_callback=custom +fun:dfsan_flush=uninstrumented +fun:dfsan_flush=discard ############################################################################### # glibc diff --git a/lib/esan/cache_frag.cpp b/lib/esan/cache_frag.cpp deleted file mode 100644 index 5fa5c7d54244..000000000000 --- a/lib/esan/cache_frag.cpp +++ /dev/null @@ -1,208 +0,0 @@ -//===-- cache_frag.cpp ----------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// This file contains cache fragmentation-specific code. -//===----------------------------------------------------------------------===// - -#include "esan.h" -#include "esan_flags.h" -#include "sanitizer_common/sanitizer_addrhashmap.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_placement_new.h" -#include <string.h> - -namespace __esan { - -//===-- Struct field access counter runtime -------------------------------===// - -// This should be kept consistent with LLVM's EfficiencySanitizer StructInfo. -struct StructInfo { - const char *StructName; - u32 Size; - u32 NumFields; - u32 *FieldOffset; // auxiliary struct field info. - u32 *FieldSize; // auxiliary struct field info. - const char **FieldTypeName; // auxiliary struct field info. - u64 *FieldCounters; - u64 *ArrayCounter; - bool hasAuxFieldInfo() { return FieldOffset != nullptr; } -}; - -// This should be kept consistent with LLVM's EfficiencySanitizer CacheFragInfo. -// The tool-specific information per compilation unit (module). -struct CacheFragInfo { - const char *UnitName; - u32 NumStructs; - StructInfo *Structs; -}; - -struct StructCounter { - StructInfo *Struct; - u64 Count; // The total access count of the struct. - u64 Ratio; // Difference ratio for the struct layout access. -}; - -// We use StructHashMap to keep track of an unique copy of StructCounter. -typedef AddrHashMap<StructCounter, 31051> StructHashMap; -struct Context { - StructHashMap StructMap; - u32 NumStructs; - u64 TotalCount; // The total access count of all structs. -}; -static Context *Ctx; - -static void reportStructSummary() { - // FIXME: provide a better struct field access summary report. - Report("%s: total struct field access count = %llu\n", SanitizerToolName, - Ctx->TotalCount); -} - -// FIXME: we are still exploring proper ways to evaluate the difference between -// struct field counts. Currently, we use a simple formula to calculate the -// difference ratio: V1/V2. -static inline u64 computeDifferenceRatio(u64 Val1, u64 Val2) { - if (Val2 > Val1) { - Swap(Val1, Val2); - } - if (Val2 == 0) - Val2 = 1; - return (Val1 / Val2); -} - -static void reportStructCounter(StructHashMap::Handle &Handle) { - const u32 TypePrintLimit = 512; - const char *type, *start, *end; - StructInfo *Struct = Handle->Struct; - // Union field address calculation is done via bitcast instead of GEP, - // so the count for union is always 0. - // We skip the union report to avoid confusion. - if (strncmp(Struct->StructName, "union.", 6) == 0) - return; - // Remove the '.' after class/struct during print. - if (strncmp(Struct->StructName, "class.", 6) == 0) { - type = "class"; - start = &Struct->StructName[6]; - } else { - type = "struct"; - start = &Struct->StructName[7]; - } - // Remove the suffixes with '$' during print. - end = strchr(start, '$'); - CHECK(end != nullptr); - Report(" %s %.*s\n", type, end - start, start); - Report(" size = %u, count = %llu, ratio = %llu, array access = %llu\n", - Struct->Size, Handle->Count, Handle->Ratio, *Struct->ArrayCounter); - if (Struct->hasAuxFieldInfo()) { - for (u32 i = 0; i < Struct->NumFields; ++i) { - Report(" #%2u: offset = %u,\t size = %u," - "\t count = %llu,\t type = %.*s\n", - i, Struct->FieldOffset[i], Struct->FieldSize[i], - Struct->FieldCounters[i], TypePrintLimit, Struct->FieldTypeName[i]); - } - } else { - for (u32 i = 0; i < Struct->NumFields; ++i) { - Report(" #%2u: count = %llu\n", i, Struct->FieldCounters[i]); - } - } -} - -static void computeStructRatio(StructHashMap::Handle &Handle) { - Handle->Ratio = 0; - Handle->Count = Handle->Struct->FieldCounters[0]; - for (u32 i = 1; i < Handle->Struct->NumFields; ++i) { - Handle->Count += Handle->Struct->FieldCounters[i]; - Handle->Ratio += computeDifferenceRatio( - Handle->Struct->FieldCounters[i - 1], Handle->Struct->FieldCounters[i]); - } - Ctx->TotalCount += Handle->Count; - if (Handle->Ratio >= (u64)getFlags()->report_threshold || - (Verbosity() >= 1 && Handle->Count > 0)) - reportStructCounter(Handle); -} - -static void registerStructInfo(CacheFragInfo *CacheFrag) { - for (u32 i = 0; i < CacheFrag->NumStructs; ++i) { - StructInfo *Struct = &CacheFrag->Structs[i]; - StructHashMap::Handle H(&Ctx->StructMap, (uptr)Struct->FieldCounters); - if (H.created()) { - VPrintf(2, " Register %s: %u fields\n", Struct->StructName, - Struct->NumFields); - H->Struct = Struct; - ++Ctx->NumStructs; - } else { - VPrintf(2, " Duplicated %s: %u fields\n", Struct->StructName, - Struct->NumFields); - } - } -} - -static void unregisterStructInfo(CacheFragInfo *CacheFrag) { - // FIXME: if the library is unloaded before finalizeCacheFrag, we should - // collect the result for later report. - for (u32 i = 0; i < CacheFrag->NumStructs; ++i) { - StructInfo *Struct = &CacheFrag->Structs[i]; - StructHashMap::Handle H(&Ctx->StructMap, (uptr)Struct->FieldCounters, true); - if (H.exists()) { - VPrintf(2, " Unregister %s: %u fields\n", Struct->StructName, - Struct->NumFields); - // FIXME: we should move this call to finalizeCacheFrag once we can - // iterate over the hash map there. - computeStructRatio(H); - --Ctx->NumStructs; - } else { - VPrintf(2, " Duplicated %s: %u fields\n", Struct->StructName, - Struct->NumFields); - } - } - static bool Reported = false; - if (Ctx->NumStructs == 0 && !Reported) { - Reported = true; - reportStructSummary(); - } -} - -//===-- Init/exit functions -----------------------------------------------===// - -void processCacheFragCompilationUnitInit(void *Ptr) { - CacheFragInfo *CacheFrag = (CacheFragInfo *)Ptr; - VPrintf(2, "in esan::%s: %s with %u class(es)/struct(s)\n", __FUNCTION__, - CacheFrag->UnitName, CacheFrag->NumStructs); - registerStructInfo(CacheFrag); -} - -void processCacheFragCompilationUnitExit(void *Ptr) { - CacheFragInfo *CacheFrag = (CacheFragInfo *)Ptr; - VPrintf(2, "in esan::%s: %s with %u class(es)/struct(s)\n", __FUNCTION__, - CacheFrag->UnitName, CacheFrag->NumStructs); - unregisterStructInfo(CacheFrag); -} - -void initializeCacheFrag() { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - // We use placement new to initialize Ctx before C++ static initializaion. - // We make CtxMem 8-byte aligned for atomic operations in AddrHashMap. - static u64 CtxMem[sizeof(Context) / sizeof(u64) + 1]; - Ctx = new (CtxMem) Context(); - Ctx->NumStructs = 0; -} - -int finalizeCacheFrag() { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - return 0; -} - -void reportCacheFrag() { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - // FIXME: Not yet implemented. We need to iterate over all of the - // compilation unit data. -} - -} // namespace __esan diff --git a/lib/esan/cache_frag.h b/lib/esan/cache_frag.h deleted file mode 100644 index 646d3f85ed97..000000000000 --- a/lib/esan/cache_frag.h +++ /dev/null @@ -1,29 +0,0 @@ -//===-- cache_frag.h --------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Header for cache-fragmentation-specific code. -//===----------------------------------------------------------------------===// - -#ifndef CACHE_FRAG_H -#define CACHE_FRAG_H - -namespace __esan { - -void processCacheFragCompilationUnitInit(void *Ptr); -void processCacheFragCompilationUnitExit(void *Ptr); - -void initializeCacheFrag(); -int finalizeCacheFrag(); -void reportCacheFrag(); - -} // namespace __esan - -#endif // CACHE_FRAG_H diff --git a/lib/esan/esan.cpp b/lib/esan/esan.cpp deleted file mode 100644 index 44b8032caa1a..000000000000 --- a/lib/esan/esan.cpp +++ /dev/null @@ -1,278 +0,0 @@ -//===-- esan.cpp ----------------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Main file (entry points) for the Esan run-time. -//===----------------------------------------------------------------------===// - -#include "esan.h" -#include "esan_flags.h" -#include "esan_interface_internal.h" -#include "esan_shadow.h" -#include "cache_frag.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_flag_parser.h" -#include "sanitizer_common/sanitizer_flags.h" -#include "working_set.h" - -// See comment below. -extern "C" { -extern void __cxa_atexit(void (*function)(void)); -} - -namespace __esan { - -bool EsanIsInitialized; -bool EsanDuringInit; -ShadowMapping Mapping; - -// Different tools use different scales within the same shadow mapping scheme. -// The scale used here must match that used by the compiler instrumentation. -// This array is indexed by the ToolType enum. -static const uptr ShadowScale[] = { - 0, // ESAN_None. - 2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2. - 6, // ESAN_WorkingSet: 64B:1B, so 64 to 1 == >>6. -}; - -// We are combining multiple performance tuning tools under the umbrella of -// one EfficiencySanitizer super-tool. Most of our tools have very similar -// memory access instrumentation, shadow memory mapping, libc interception, -// etc., and there is typically more shared code than distinct code. -// -// We are not willing to dispatch on tool dynamically in our fastpath -// instrumentation: thus, which tool to use is a static option selected -// at compile time and passed to __esan_init(). -// -// We are willing to pay the overhead of tool dispatch in the slowpath to more -// easily share code. We expect to only come here rarely. -// If this becomes a performance hit, we can add separate interface -// routines for each subtool (e.g., __esan_cache_frag_aligned_load_4). -// But for libc interceptors, we'll have to do one of the following: -// A) Add multiple-include support to sanitizer_common_interceptors.inc, -// instantiate it separately for each tool, and call the selected -// tool's intercept setup code. -// B) Build separate static runtime libraries, one for each tool. -// C) Completely split the tools into separate sanitizers. - -void processRangeAccess(uptr PC, uptr Addr, int Size, bool IsWrite) { - VPrintf(3, "in esan::%s %p: %c %p %d\n", __FUNCTION__, PC, - IsWrite ? 'w' : 'r', Addr, Size); - if (__esan_which_tool == ESAN_CacheFrag) { - // TODO(bruening): add shadow mapping and update shadow bits here. - // We'll move this to cache_frag.cpp once we have something. - } else if (__esan_which_tool == ESAN_WorkingSet) { - processRangeAccessWorkingSet(PC, Addr, Size, IsWrite); - } -} - -bool processSignal(int SigNum, void (*Handler)(int), void (**Result)(int)) { - if (__esan_which_tool == ESAN_WorkingSet) - return processWorkingSetSignal(SigNum, Handler, Result); - return true; -} - -bool processSigaction(int SigNum, const void *Act, void *OldAct) { - if (__esan_which_tool == ESAN_WorkingSet) - return processWorkingSetSigaction(SigNum, Act, OldAct); - return true; -} - -bool processSigprocmask(int How, void *Set, void *OldSet) { - if (__esan_which_tool == ESAN_WorkingSet) - return processWorkingSetSigprocmask(How, Set, OldSet); - return true; -} - -#if SANITIZER_DEBUG -static bool verifyShadowScheme() { - // Sanity checks for our shadow mapping scheme. - uptr AppStart, AppEnd; - if (Verbosity() >= 3) { - for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) { - VPrintf(3, "App #%d: [%zx-%zx) (%zuGB)\n", i, AppStart, AppEnd, - (AppEnd - AppStart) >> 30); - } - } - for (int Scale = 0; Scale < 8; ++Scale) { - Mapping.initialize(Scale); - if (Verbosity() >= 3) { - VPrintf(3, "\nChecking scale %d\n", Scale); - uptr ShadowStart, ShadowEnd; - for (int i = 0; getShadowRegion(i, &ShadowStart, &ShadowEnd); ++i) { - VPrintf(3, "Shadow #%d: [%zx-%zx) (%zuGB)\n", i, ShadowStart, - ShadowEnd, (ShadowEnd - ShadowStart) >> 30); - } - for (int i = 0; getShadowRegion(i, &ShadowStart, &ShadowEnd); ++i) { - VPrintf(3, "Shadow(Shadow) #%d: [%zx-%zx)\n", i, - appToShadow(ShadowStart), appToShadow(ShadowEnd - 1)+1); - } - } - for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) { - DCHECK(isAppMem(AppStart)); - DCHECK(!isAppMem(AppStart - 1)); - DCHECK(isAppMem(AppEnd - 1)); - DCHECK(!isAppMem(AppEnd)); - DCHECK(!isShadowMem(AppStart)); - DCHECK(!isShadowMem(AppEnd - 1)); - DCHECK(isShadowMem(appToShadow(AppStart))); - DCHECK(isShadowMem(appToShadow(AppEnd - 1))); - // Double-shadow checks. - DCHECK(!isShadowMem(appToShadow(appToShadow(AppStart)))); - DCHECK(!isShadowMem(appToShadow(appToShadow(AppEnd - 1)))); - } - // Ensure no shadow regions overlap each other. - uptr ShadowAStart, ShadowBStart, ShadowAEnd, ShadowBEnd; - for (int i = 0; getShadowRegion(i, &ShadowAStart, &ShadowAEnd); ++i) { - for (int j = 0; getShadowRegion(j, &ShadowBStart, &ShadowBEnd); ++j) { - DCHECK(i == j || ShadowAStart >= ShadowBEnd || - ShadowAEnd <= ShadowBStart); - } - } - } - return true; -} -#endif - -uptr VmaSize; - -static void initializeShadow() { - verifyAddressSpace(); - - // This is based on the assumption that the intial stack is always allocated - // in the topmost segment of the user address space and the assumption - // holds true on all the platforms currently supported. - VmaSize = - (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); - - DCHECK(verifyShadowScheme()); - - Mapping.initialize(ShadowScale[__esan_which_tool]); - - VPrintf(1, "Shadow scale=%d offset=%p\n", Mapping.Scale, Mapping.Offset); - - uptr ShadowStart, ShadowEnd; - for (int i = 0; getShadowRegion(i, &ShadowStart, &ShadowEnd); ++i) { - VPrintf(1, "Shadow #%d: [%zx-%zx) (%zuGB)\n", i, ShadowStart, ShadowEnd, - (ShadowEnd - ShadowStart) >> 30); - - uptr Map = 0; - if (__esan_which_tool == ESAN_WorkingSet) { - // We want to identify all shadow pages that are touched so we start - // out inaccessible. - Map = (uptr)MmapFixedNoAccess(ShadowStart, ShadowEnd- ShadowStart, - "shadow"); - } else { - if (MmapFixedNoReserve(ShadowStart, ShadowEnd - ShadowStart, "shadow")) - Map = ShadowStart; - } - if (Map != ShadowStart) { - Printf("FATAL: EfficiencySanitizer failed to map its shadow memory.\n"); - Die(); - } - - if (common_flags()->no_huge_pages_for_shadow) - NoHugePagesInRegion(ShadowStart, ShadowEnd - ShadowStart); - if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(ShadowStart, ShadowEnd - ShadowStart); - - // TODO: Call MmapNoAccess() on in-between regions. - } -} - -void initializeLibrary(ToolType Tool) { - // We assume there is only one thread during init, but we need to - // guard against double-init when we're (re-)called from an - // early interceptor. - if (EsanIsInitialized || EsanDuringInit) - return; - EsanDuringInit = true; - CHECK(Tool == __esan_which_tool); - SanitizerToolName = "EfficiencySanitizer"; - CacheBinaryName(); - initializeFlags(); - - // Intercepting libc _exit or exit via COMMON_INTERCEPTOR_ON_EXIT only - // finalizes on an explicit exit call by the app. To handle a normal - // exit we register an atexit handler. - ::__cxa_atexit((void (*)())finalizeLibrary); - - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool <= ESAN_None || __esan_which_tool >= ESAN_Max) { - Printf("ERROR: unknown tool %d requested\n", __esan_which_tool); - Die(); - } - - initializeShadow(); - if (__esan_which_tool == ESAN_WorkingSet) - initializeShadowWorkingSet(); - - initializeInterceptors(); - - if (__esan_which_tool == ESAN_CacheFrag) { - initializeCacheFrag(); - } else if (__esan_which_tool == ESAN_WorkingSet) { - initializeWorkingSet(); - } - - EsanIsInitialized = true; - EsanDuringInit = false; -} - -int finalizeLibrary() { - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - return finalizeCacheFrag(); - } else if (__esan_which_tool == ESAN_WorkingSet) { - return finalizeWorkingSet(); - } - return 0; -} - -void reportResults() { - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - return reportCacheFrag(); - } else if (__esan_which_tool == ESAN_WorkingSet) { - return reportWorkingSet(); - } -} - -void processCompilationUnitInit(void *Ptr) { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - DCHECK(Ptr != nullptr); - processCacheFragCompilationUnitInit(Ptr); - } else { - DCHECK(Ptr == nullptr); - } -} - -// This is called when the containing module is unloaded. -// For the main executable module, this is called after finalizeLibrary. -void processCompilationUnitExit(void *Ptr) { - VPrintf(2, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_CacheFrag) { - DCHECK(Ptr != nullptr); - processCacheFragCompilationUnitExit(Ptr); - } else { - DCHECK(Ptr == nullptr); - } -} - -unsigned int getSampleCount() { - VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (__esan_which_tool == ESAN_WorkingSet) { - return getSampleCountWorkingSet(); - } - return 0; -} - -} // namespace __esan diff --git a/lib/esan/esan.h b/lib/esan/esan.h deleted file mode 100644 index e73b21e56ff1..000000000000 --- a/lib/esan/esan.h +++ /dev/null @@ -1,61 +0,0 @@ -//===-- esan.h --------------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Main internal esan header file. -// -// Ground rules: -// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static -// function-scope locals) -// - All functions/classes/etc reside in namespace __esan, except for those -// declared in esan_interface_internal.h. -// - Platform-specific files should be used instead of ifdefs (*). -// - No system headers included in header files (*). -// - Platform specific headers included only into platform-specific files (*). -// -// (*) Except when inlining is critical for performance. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_H -#define ESAN_H - -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" -#include "esan_interface_internal.h" - -namespace __esan { - -extern bool EsanIsInitialized; -extern bool EsanDuringInit; -extern uptr VmaSize; - -void initializeLibrary(ToolType Tool); -int finalizeLibrary(); -void reportResults(); -unsigned int getSampleCount(); -// Esan creates the variable per tool per compilation unit at compile time -// and passes its pointer Ptr to the runtime library. -void processCompilationUnitInit(void *Ptr); -void processCompilationUnitExit(void *Ptr); -void processRangeAccess(uptr PC, uptr Addr, int Size, bool IsWrite); -void initializeInterceptors(); - -// Platform-dependent routines. -void verifyAddressSpace(); -bool fixMmapAddr(void **Addr, SIZE_T Size, int Flags); -uptr checkMmapResult(uptr Addr, SIZE_T Size); -// The return value indicates whether to call the real version or not. -bool processSignal(int SigNum, void (*Handler)(int), void (**Result)(int)); -bool processSigaction(int SigNum, const void *Act, void *OldAct); -bool processSigprocmask(int How, void *Set, void *OldSet); - -} // namespace __esan - -#endif // ESAN_H diff --git a/lib/esan/esan.syms.extra b/lib/esan/esan.syms.extra deleted file mode 100644 index d6397d4c350f..000000000000 --- a/lib/esan/esan.syms.extra +++ /dev/null @@ -1,4 +0,0 @@ -__esan_init -__esan_exit -__esan_aligned* -__esan_unaligned* diff --git a/lib/esan/esan_circular_buffer.h b/lib/esan/esan_circular_buffer.h deleted file mode 100644 index 9ce102d04d8f..000000000000 --- a/lib/esan/esan_circular_buffer.h +++ /dev/null @@ -1,96 +0,0 @@ -//===-- esan_circular_buffer.h ----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Circular buffer data structure. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_common.h" - -namespace __esan { - -// A circular buffer for POD data whose memory is allocated using mmap. -// There are two usage models: one is to use initialize/free (for global -// instances) and the other is to use placement new with the -// constructor and to call the destructor or free (they are equivalent). -template<typename T> -class CircularBuffer { - public: - // To support global instances we cannot initialize any field in the - // default constructor. - explicit CircularBuffer() {} - CircularBuffer(uptr BufferCapacity) { - initialize(BufferCapacity); - WasConstructed = true; - } - ~CircularBuffer() { - if (WasConstructed) // Else caller will call free() explicitly. - free(); - } - void initialize(uptr BufferCapacity) { - Capacity = BufferCapacity; - // MmapOrDie rounds up to the page size for us. - Data = (T *)MmapOrDie(Capacity * sizeof(T), "CircularBuffer"); - StartIdx = 0; - Count = 0; - WasConstructed = false; - } - void free() { - UnmapOrDie(Data, Capacity * sizeof(T)); - } - T &operator[](uptr Idx) { - CHECK_LT(Idx, Count); - uptr ArrayIdx = (StartIdx + Idx) % Capacity; - return Data[ArrayIdx]; - } - const T &operator[](uptr Idx) const { - CHECK_LT(Idx, Count); - uptr ArrayIdx = (StartIdx + Idx) % Capacity; - return Data[ArrayIdx]; - } - void push_back(const T &Item) { - CHECK_GT(Capacity, 0); - uptr ArrayIdx = (StartIdx + Count) % Capacity; - Data[ArrayIdx] = Item; - if (Count < Capacity) - ++Count; - else - StartIdx = (StartIdx + 1) % Capacity; - } - T &back() { - CHECK_GT(Count, 0); - uptr ArrayIdx = (StartIdx + Count - 1) % Capacity; - return Data[ArrayIdx]; - } - void pop_back() { - CHECK_GT(Count, 0); - --Count; - } - uptr size() const { - return Count; - } - void clear() { - StartIdx = 0; - Count = 0; - } - bool empty() const { return size() == 0; } - - private: - CircularBuffer(const CircularBuffer&); - void operator=(const CircularBuffer&); - - bool WasConstructed; - T *Data; - uptr Capacity; - uptr StartIdx; - uptr Count; -}; - -} // namespace __esan diff --git a/lib/esan/esan_flags.cpp b/lib/esan/esan_flags.cpp deleted file mode 100644 index c90bf2493f7b..000000000000 --- a/lib/esan/esan_flags.cpp +++ /dev/null @@ -1,60 +0,0 @@ -//===-- esan_flags.cc -------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Esan flag parsing logic. -//===----------------------------------------------------------------------===// - -#include "esan_flags.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_flag_parser.h" -#include "sanitizer_common/sanitizer_flags.h" - -using namespace __sanitizer; - -namespace __esan { - -static const char EsanOptsEnv[] = "ESAN_OPTIONS"; - -Flags EsanFlagsDontUseDirectly; - -void Flags::setDefaults() { -#define ESAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; -#include "esan_flags.inc" -#undef ESAN_FLAG -} - -static void registerEsanFlags(FlagParser *Parser, Flags *F) { -#define ESAN_FLAG(Type, Name, DefaultValue, Description) \ - RegisterFlag(Parser, #Name, Description, &F->Name); -#include "esan_flags.inc" -#undef ESAN_FLAG -} - -void initializeFlags() { - SetCommonFlagsDefaults(); - Flags *F = getFlags(); - F->setDefaults(); - - FlagParser Parser; - registerEsanFlags(&Parser, F); - RegisterCommonFlags(&Parser); - Parser.ParseString(GetEnv(EsanOptsEnv)); - - InitializeCommonFlags(); - if (Verbosity()) - ReportUnrecognizedFlags(); - if (common_flags()->help) - Parser.PrintFlagDescriptions(); - - __sanitizer_set_report_path(common_flags()->log_path); -} - -} // namespace __esan diff --git a/lib/esan/esan_flags.h b/lib/esan/esan_flags.h deleted file mode 100644 index c8f4ef5ab2b5..000000000000 --- a/lib/esan/esan_flags.h +++ /dev/null @@ -1,41 +0,0 @@ -//===-- esan_flags.h --------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Esan runtime flags. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_FLAGS_H -#define ESAN_FLAGS_H - -#include "sanitizer_common/sanitizer_internal_defs.h" -#include "sanitizer_common/sanitizer_flag_parser.h" - -namespace __esan { - -class Flags { -public: -#define ESAN_FLAG(Type, Name, DefaultValue, Description) Type Name; -#include "esan_flags.inc" -#undef ESAN_FLAG - - void setDefaults(); -}; - -extern Flags EsanFlagsDontUseDirectly; -inline Flags *getFlags() { - return &EsanFlagsDontUseDirectly; -} - -void initializeFlags(); - -} // namespace __esan - -#endif // ESAN_FLAGS_H diff --git a/lib/esan/esan_flags.inc b/lib/esan/esan_flags.inc deleted file mode 100644 index 5687caca2989..000000000000 --- a/lib/esan/esan_flags.inc +++ /dev/null @@ -1,56 +0,0 @@ -//===-- esan_flags.inc ------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Esan runtime flags. -// -//===----------------------------------------------------------------------===// - -#ifndef ESAN_FLAG -# error "Define ESAN_FLAG prior to including this file!" -#endif - -// ESAN_FLAG(Type, Name, DefaultValue, Description) -// See COMMON_FLAG in sanitizer_flags.inc for more details. - -//===----------------------------------------------------------------------===// -// Cross-tool options -//===----------------------------------------------------------------------===// - -ESAN_FLAG(int, cache_line_size, 64, - "The number of bytes in a cache line. For the working-set tool, this " - "cannot be changed without also changing the compiler " - "instrumentation.") - -//===----------------------------------------------------------------------===// -// Working set tool options -//===----------------------------------------------------------------------===// - -ESAN_FLAG(bool, record_snapshots, true, - "Working set tool: whether to sample snapshots during a run.") - -// Typical profiling uses a 10ms timer. Our snapshots take some work -// to scan memory so we reduce to 20ms. -// To disable samples, turn off record_snapshots. -ESAN_FLAG(int, sample_freq, 20, - "Working set tool: sampling frequency in milliseconds.") - -// This controls the difference in frequency between each successive series -// of snapshots. There are 8 in total, with number 0 using sample_freq. -// Number N samples number N-1 every (1 << snapshot_step) instance of N-1. -ESAN_FLAG(int, snapshot_step, 2, "Working set tool: the log of the sampling " - "performed for the next-higher-frequency snapshot series.") - -//===----------------------------------------------------------------------===// -// Cache Fragmentation tool options -//===----------------------------------------------------------------------===// - -// The difference information of a struct is reported if the struct's difference -// score is greater than the report_threshold. -ESAN_FLAG(int, report_threshold, 1<<10, "Cache-frag tool: the struct difference" - " score threshold for reporting.") diff --git a/lib/esan/esan_hashtable.h b/lib/esan/esan_hashtable.h deleted file mode 100644 index 7bd829740400..000000000000 --- a/lib/esan/esan_hashtable.h +++ /dev/null @@ -1,381 +0,0 @@ -//===-- esan_hashtable.h ----------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Generic resizing hashtable. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_allocator_internal.h" -#include "sanitizer_common/sanitizer_internal_defs.h" -#include "sanitizer_common/sanitizer_mutex.h" -#include <stddef.h> - -namespace __esan { - -//===----------------------------------------------------------------------===// -// Default hash and comparison functions -//===----------------------------------------------------------------------===// - -template <typename T> struct DefaultHash { - size_t operator()(const T &Key) const { - return (size_t)Key; - } -}; - -template <typename T> struct DefaultEqual { - bool operator()(const T &Key1, const T &Key2) const { - return Key1 == Key2; - } -}; - -//===----------------------------------------------------------------------===// -// HashTable declaration -//===----------------------------------------------------------------------===// - -// A simple resizing and mutex-locked hashtable. -// -// If the default hash functor is used, KeyTy must have an operator size_t(). -// If the default comparison functor is used, KeyTy must have an operator ==. -// -// By default all operations are internally-synchronized with a mutex, with no -// synchronization for payloads once hashtable functions return. If -// ExternalLock is set to true, the caller should call the lock() and unlock() -// routines around all hashtable operations and subsequent manipulation of -// payloads. -template <typename KeyTy, typename DataTy, bool ExternalLock = false, - typename HashFuncTy = DefaultHash<KeyTy>, - typename EqualFuncTy = DefaultEqual<KeyTy> > -class HashTable { -public: - // InitialCapacity must be a power of 2. - // ResizeFactor must be between 1 and 99 and indicates the - // maximum percentage full that the table should ever be. - HashTable(u32 InitialCapacity = 2048, u32 ResizeFactor = 70); - ~HashTable(); - bool lookup(const KeyTy &Key, DataTy &Payload); // Const except for Mutex. - bool add(const KeyTy &Key, const DataTy &Payload); - bool remove(const KeyTy &Key); - u32 size(); // Const except for Mutex. - // If the table is internally-synchronized, this lock must not be held - // while a hashtable function is called as it will deadlock: the lock - // is not recursive. This is meant for use with externally-synchronized - // tables or with an iterator. - void lock(); - void unlock(); - -private: - struct HashEntry { - KeyTy Key; - DataTy Payload; - HashEntry *Next; - }; - -public: - struct HashPair { - HashPair(KeyTy Key, DataTy Data) : Key(Key), Data(Data) {} - KeyTy Key; - DataTy Data; - }; - - // This iterator does not perform any synchronization. - // It expects the caller to lock the table across the whole iteration. - // Calling HashTable functions while using the iterator is not supported. - // The iterator returns copies of the keys and data. - class iterator { - public: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table); - iterator(const iterator &Src) = default; - iterator &operator=(const iterator &Src) = default; - HashPair operator*(); - iterator &operator++(); - iterator &operator++(int); - bool operator==(const iterator &Cmp) const; - bool operator!=(const iterator &Cmp) const; - - private: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table, - int Idx); - friend HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>; - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table; - int Idx; - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::HashEntry - *Entry; - }; - - // No erase or insert iterator supported - iterator begin(); - iterator end(); - -private: - void resize(); - - HashEntry **Table; - u32 Capacity; - u32 Entries; - const u32 ResizeFactor; - BlockingMutex Mutex; - const HashFuncTy HashFunc; - const EqualFuncTy EqualFunc; -}; - -//===----------------------------------------------------------------------===// -// Hashtable implementation -//===----------------------------------------------------------------------===// - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::HashTable( - u32 InitialCapacity, u32 ResizeFactor) - : Capacity(InitialCapacity), Entries(0), ResizeFactor(ResizeFactor), - HashFunc(HashFuncTy()), EqualFunc(EqualFuncTy()) { - CHECK(IsPowerOfTwo(Capacity)); - CHECK(ResizeFactor >= 1 && ResizeFactor <= 99); - Table = (HashEntry **)InternalAlloc(Capacity * sizeof(HashEntry *)); - internal_memset(Table, 0, Capacity * sizeof(HashEntry *)); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::~HashTable() { - for (u32 i = 0; i < Capacity; ++i) { - HashEntry *Entry = Table[i]; - while (Entry != nullptr) { - HashEntry *Next = Entry->Next; - Entry->Payload.~DataTy(); - InternalFree(Entry); - Entry = Next; - } - } - InternalFree(Table); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -u32 HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::size() { - u32 Res; - if (!ExternalLock) - Mutex.Lock(); - Res = Entries; - if (!ExternalLock) - Mutex.Unlock(); - return Res; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::lookup( - const KeyTy &Key, DataTy &Payload) { - if (!ExternalLock) - Mutex.Lock(); - bool Found = false; - size_t Hash = HashFunc(Key) % Capacity; - HashEntry *Entry = Table[Hash]; - for (; Entry != nullptr; Entry = Entry->Next) { - if (EqualFunc(Entry->Key, Key)) { - Payload = Entry->Payload; - Found = true; - break; - } - } - if (!ExternalLock) - Mutex.Unlock(); - return Found; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -void HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::resize() { - if (!ExternalLock) - Mutex.CheckLocked(); - size_t OldCapacity = Capacity; - HashEntry **OldTable = Table; - Capacity *= 2; - Table = (HashEntry **)InternalAlloc(Capacity * sizeof(HashEntry *)); - internal_memset(Table, 0, Capacity * sizeof(HashEntry *)); - // Re-hash - for (u32 i = 0; i < OldCapacity; ++i) { - HashEntry *OldEntry = OldTable[i]; - while (OldEntry != nullptr) { - HashEntry *Next = OldEntry->Next; - size_t Hash = HashFunc(OldEntry->Key) % Capacity; - OldEntry->Next = Table[Hash]; - Table[Hash] = OldEntry; - OldEntry = Next; - } - } -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::add( - const KeyTy &Key, const DataTy &Payload) { - if (!ExternalLock) - Mutex.Lock(); - bool Exists = false; - size_t Hash = HashFunc(Key) % Capacity; - HashEntry *Entry = Table[Hash]; - for (; Entry != nullptr; Entry = Entry->Next) { - if (EqualFunc(Entry->Key, Key)) { - Exists = true; - break; - } - } - if (!Exists) { - Entries++; - if (Entries * 100 >= Capacity * ResizeFactor) { - resize(); - Hash = HashFunc(Key) % Capacity; - } - HashEntry *Add = (HashEntry *)InternalAlloc(sizeof(*Add)); - Add->Key = Key; - Add->Payload = Payload; - Add->Next = Table[Hash]; - Table[Hash] = Add; - } - if (!ExternalLock) - Mutex.Unlock(); - return !Exists; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::remove( - const KeyTy &Key) { - if (!ExternalLock) - Mutex.Lock(); - bool Found = false; - size_t Hash = HashFunc(Key) % Capacity; - HashEntry *Entry = Table[Hash]; - HashEntry *Prev = nullptr; - for (; Entry != nullptr; Prev = Entry, Entry = Entry->Next) { - if (EqualFunc(Entry->Key, Key)) { - Found = true; - Entries--; - if (Prev == nullptr) - Table[Hash] = Entry->Next; - else - Prev->Next = Entry->Next; - Entry->Payload.~DataTy(); - InternalFree(Entry); - break; - } - } - if (!ExternalLock) - Mutex.Unlock(); - return Found; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -void HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::lock() { - Mutex.Lock(); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -void HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::unlock() { - Mutex.Unlock(); -} - -//===----------------------------------------------------------------------===// -// Iterator implementation -//===----------------------------------------------------------------------===// - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table) - : Table(Table), Idx(-1), Entry(nullptr) { - operator++(); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - iterator( - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy> *Table, - int Idx) - : Table(Table), Idx(Idx), Entry(nullptr) { - CHECK(Idx >= (int)Table->Capacity); // Only used to create end(). -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::HashPair - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - operator*() { - CHECK(Idx >= 0 && Idx < (int)Table->Capacity); - CHECK(Entry != nullptr); - return HashPair(Entry->Key, Entry->Payload); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator & - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - operator++() { - if (Entry != nullptr) - Entry = Entry->Next; - while (Entry == nullptr) { - ++Idx; - if (Idx >= (int)Table->Capacity) - break; // At end(). - Entry = Table->Table[Idx]; - } - return *this; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator & - HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: - operator++(int) { - iterator Temp(*this); - operator++(); - return Temp; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: -operator==(const iterator &Cmp) const { - return Cmp.Table == Table && Cmp.Idx == Idx && Cmp.Entry == Entry; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -bool HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::iterator:: -operator!=(const iterator &Cmp) const { - return Cmp.Table != Table || Cmp.Idx != Idx || Cmp.Entry != Entry; -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::begin() { - return iterator(this); -} - -template <typename KeyTy, typename DataTy, bool ExternalLock, - typename HashFuncTy, typename EqualFuncTy> -typename HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, - EqualFuncTy>::iterator -HashTable<KeyTy, DataTy, ExternalLock, HashFuncTy, EqualFuncTy>::end() { - return iterator(this, Capacity); -} - -} // namespace __esan diff --git a/lib/esan/esan_interceptors.cpp b/lib/esan/esan_interceptors.cpp deleted file mode 100644 index 833faa2cd5bc..000000000000 --- a/lib/esan/esan_interceptors.cpp +++ /dev/null @@ -1,512 +0,0 @@ -//===-- esan_interceptors.cpp ---------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Interception routines for the esan run-time. -//===----------------------------------------------------------------------===// - -#include "esan.h" -#include "esan_shadow.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_libc.h" -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_stacktrace.h" - -using namespace __esan; // NOLINT - -#define CUR_PC() (StackTrace::GetCurrentPc()) - -//===----------------------------------------------------------------------===// -// Interception via sanitizer common interceptors -//===----------------------------------------------------------------------===// - -// Get the per-platform defines for what is possible to intercept -#include "sanitizer_common/sanitizer_platform_interceptors.h" - -DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr) - -// TODO(bruening): tsan disables several interceptors (getpwent, etc.) claiming -// that interception is a perf hit: should we do the same? - -// We have no need to intercept: -#undef SANITIZER_INTERCEPT_TLS_GET_ADDR - -// TODO(bruening): the common realpath interceptor assumes malloc is -// intercepted! We should try to parametrize that, though we'll -// intercept malloc soon ourselves and can then remove this undef. -#undef SANITIZER_INTERCEPT_REALPATH - -// We provide our own version: -#undef SANITIZER_INTERCEPT_SIGPROCMASK - -#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!EsanIsInitialized) - -#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) -#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ - INTERCEPT_FUNCTION_VER(name, ver) - -// We must initialize during early interceptors, to support tcmalloc. -// This means that for some apps we fully initialize prior to -// __esan_init() being called. -// We currently do not use ctx. -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - do { \ - if (UNLIKELY(COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)) { \ - if (!UNLIKELY(EsanDuringInit)) \ - initializeLibrary(__esan_which_tool); \ - return REAL(func)(__VA_ARGS__); \ - } \ - ctx = nullptr; \ - (void)ctx; \ - } while (false) - -#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ - COMMON_INTERCEPTOR_ENTER(ctx, func, __VA_ARGS__) - -#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ - processRangeAccess(CUR_PC(), (uptr)ptr, size, true) - -#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ - processRangeAccess(CUR_PC(), (uptr)ptr, size, false) - -// This is only called if the app explicitly calls exit(), not on -// a normal exit. -#define COMMON_INTERCEPTOR_ON_EXIT(ctx) finalizeLibrary() - -#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ - do { \ - (void)(ctx); \ - (void)(file); \ - (void)(path); \ - } while (false) -#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ - do { \ - (void)(ctx); \ - (void)(file); \ - } while (false) -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ - do { \ - (void)(filename); \ - (void)(handle); \ - } while (false) -#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ - do { \ - (void)(ctx); \ - (void)(u); \ - } while (false) -#define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ - do { \ - (void)(ctx); \ - (void)(u); \ - } while (false) -#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ - do { \ - (void)(ctx); \ - (void)(path); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - } while (false) -#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ - do { \ - (void)(ctx); \ - (void)(fd); \ - (void)(newfd); \ - } while (false) -#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ - do { \ - (void)(ctx); \ - (void)(name); \ - } while (false) -#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ - do { \ - (void)(ctx); \ - (void)(thread); \ - (void)(name); \ - } while (false) -#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) -#define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \ - do { \ - (void)(ctx); \ - (void)(m); \ - } while (false) -#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ - do { \ - (void)(ctx); \ - (void)(m); \ - } while (false) -#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ - do { \ - (void)(ctx); \ - (void)(m); \ - } while (false) -#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ - do { \ - (void)(ctx); \ - (void)(msg); \ - } while (false) -#define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ - do { \ - } while (false) - -#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ - off) \ - do { \ - if (!fixMmapAddr(&addr, sz, flags)) \ - return (void *)-1; \ - void *result = REAL(mmap)(addr, sz, prot, flags, fd, off); \ - return (void *)checkMmapResult((uptr)result, sz); \ - } while (false) - -#include "sanitizer_common/sanitizer_common_interceptors.inc" - -//===----------------------------------------------------------------------===// -// Syscall interception -//===----------------------------------------------------------------------===// - -// We want the caller's PC b/c unlike the other function interceptors these -// are separate pre and post functions called around the app's syscall(). - -#define COMMON_SYSCALL_PRE_READ_RANGE(ptr, size) \ - processRangeAccess(GET_CALLER_PC(), (uptr)ptr, size, false) - -#define COMMON_SYSCALL_PRE_WRITE_RANGE(ptr, size) \ - do { \ - (void)(ptr); \ - (void)(size); \ - } while (false) - -#define COMMON_SYSCALL_POST_READ_RANGE(ptr, size) \ - do { \ - (void)(ptr); \ - (void)(size); \ - } while (false) - -// The actual amount written is in post, not pre. -#define COMMON_SYSCALL_POST_WRITE_RANGE(ptr, size) \ - processRangeAccess(GET_CALLER_PC(), (uptr)ptr, size, true) - -#define COMMON_SYSCALL_ACQUIRE(addr) \ - do { \ - (void)(addr); \ - } while (false) -#define COMMON_SYSCALL_RELEASE(addr) \ - do { \ - (void)(addr); \ - } while (false) -#define COMMON_SYSCALL_FD_CLOSE(fd) \ - do { \ - (void)(fd); \ - } while (false) -#define COMMON_SYSCALL_FD_ACQUIRE(fd) \ - do { \ - (void)(fd); \ - } while (false) -#define COMMON_SYSCALL_FD_RELEASE(fd) \ - do { \ - (void)(fd); \ - } while (false) -#define COMMON_SYSCALL_PRE_FORK() \ - do { \ - } while (false) -#define COMMON_SYSCALL_POST_FORK(res) \ - do { \ - (void)(res); \ - } while (false) - -#include "sanitizer_common/sanitizer_common_syscalls.inc" -#include "sanitizer_common/sanitizer_syscalls_netbsd.inc" - -//===----------------------------------------------------------------------===// -// Custom interceptors -//===----------------------------------------------------------------------===// - -// TODO(bruening): move more of these to the common interception pool as they -// are shared with tsan and asan. -// While our other files match LLVM style, here we match sanitizer style as we -// expect to move these to the common pool. - -INTERCEPTOR(char *, strcpy, char *dst, const char *src) { // NOLINT - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, strcpy, dst, src); - uptr srclen = internal_strlen(src); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, srclen + 1); - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, srclen + 1); - return REAL(strcpy)(dst, src); // NOLINT -} - -INTERCEPTOR(char *, strncpy, char *dst, char *src, uptr n) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, strncpy, dst, src, n); - uptr srclen = internal_strnlen(src, n); - uptr copied_size = srclen + 1 > n ? n : srclen + 1; - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, copied_size); - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, copied_size); - return REAL(strncpy)(dst, src, n); -} - -INTERCEPTOR(int, open, const char *name, int flags, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, open, name, flags, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(open)(name, flags, mode); -} - -#if SANITIZER_LINUX -INTERCEPTOR(int, open64, const char *name, int flags, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, open64, name, flags, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(open64)(name, flags, mode); -} -#define ESAN_MAYBE_INTERCEPT_OPEN64 INTERCEPT_FUNCTION(open64) -#else -#define ESAN_MAYBE_INTERCEPT_OPEN64 -#endif - -INTERCEPTOR(int, creat, const char *name, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, creat, name, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(creat)(name, mode); -} - -#if SANITIZER_LINUX -INTERCEPTOR(int, creat64, const char *name, int mode) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, creat64, name, mode); - COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); - return REAL(creat64)(name, mode); -} -#define ESAN_MAYBE_INTERCEPT_CREAT64 INTERCEPT_FUNCTION(creat64) -#else -#define ESAN_MAYBE_INTERCEPT_CREAT64 -#endif - -INTERCEPTOR(int, unlink, char *path) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, unlink, path); - COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); - return REAL(unlink)(path); -} - -INTERCEPTOR(int, rmdir, char *path) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, rmdir, path); - COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); - return REAL(rmdir)(path); -} - -//===----------------------------------------------------------------------===// -// Signal-related interceptors -//===----------------------------------------------------------------------===// - -#if SANITIZER_LINUX || SANITIZER_FREEBSD -typedef void (*signal_handler_t)(int); -INTERCEPTOR(signal_handler_t, signal, int signum, signal_handler_t handler) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, signal, signum, handler); - signal_handler_t result; - if (!processSignal(signum, handler, &result)) - return result; - else - return REAL(signal)(signum, handler); -} -#define ESAN_MAYBE_INTERCEPT_SIGNAL INTERCEPT_FUNCTION(signal) -#else -#error Platform not supported -#define ESAN_MAYBE_INTERCEPT_SIGNAL -#endif - -#if SANITIZER_LINUX || SANITIZER_FREEBSD -DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) -INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, sigaction, signum, act, oldact); - if (!processSigaction(signum, act, oldact)) - return 0; - else - return REAL(sigaction)(signum, act, oldact); -} - -// This is required to properly use internal_sigaction. -namespace __sanitizer { -int real_sigaction(int signum, const void *act, void *oldact) { - if (REAL(sigaction) == nullptr) { - // With an instrumented allocator, this is called during interceptor init - // and we need a raw syscall solution. -#if SANITIZER_LINUX - return internal_sigaction_syscall(signum, act, oldact); -#else - return internal_sigaction(signum, act, oldact); -#endif - } - return REAL(sigaction)(signum, (const struct sigaction *)act, - (struct sigaction *)oldact); -} -} // namespace __sanitizer - -#define ESAN_MAYBE_INTERCEPT_SIGACTION INTERCEPT_FUNCTION(sigaction) -#else -#error Platform not supported -#define ESAN_MAYBE_INTERCEPT_SIGACTION -#endif - -#if SANITIZER_LINUX || SANITIZER_FREEBSD -INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, - __sanitizer_sigset_t *oldset) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, sigprocmask, how, set, oldset); - int res = 0; - if (processSigprocmask(how, set, oldset)) - res = REAL(sigprocmask)(how, set, oldset); - if (!res && oldset) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); - return res; -} -#define ESAN_MAYBE_INTERCEPT_SIGPROCMASK INTERCEPT_FUNCTION(sigprocmask) -#else -#define ESAN_MAYBE_INTERCEPT_SIGPROCMASK -#endif - -#if !SANITIZER_WINDOWS -INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set, - __sanitizer_sigset_t *oldset) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, pthread_sigmask, how, set, oldset); - int res = 0; - if (processSigprocmask(how, set, oldset)) - res = REAL(sigprocmask)(how, set, oldset); - if (!res && oldset) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); - return res; -} -#define ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK INTERCEPT_FUNCTION(pthread_sigmask) -#else -#define ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK -#endif - -//===----------------------------------------------------------------------===// -// Malloc interceptors -//===----------------------------------------------------------------------===// - -static const uptr early_alloc_buf_size = 4096; -static uptr allocated_bytes; -static char early_alloc_buf[early_alloc_buf_size]; - -static bool isInEarlyAllocBuf(const void *ptr) { - return ((uptr)ptr >= (uptr)early_alloc_buf && - ((uptr)ptr - (uptr)early_alloc_buf) < sizeof(early_alloc_buf)); -} - -static void *handleEarlyAlloc(uptr size) { - // If esan is initialized during an interceptor (which happens with some - // tcmalloc implementations that call pthread_mutex_lock), the call from - // dlsym to calloc will deadlock. - // dlsym may also call malloc before REAL(malloc) is retrieved from dlsym. - // We work around it by using a static buffer for the early malloc/calloc - // requests. - // This solution will also allow us to deliberately intercept malloc & family - // in the future (to perform tool actions on each allocation, without - // replacing the allocator), as it also solves the problem of intercepting - // calloc when it will itself be called before its REAL pointer is - // initialized. - // We do not handle multiple threads here. This only happens at process init - // time, and while it's possible for a shared library to create early threads - // that race here, we consider that to be a corner case extreme enough that - // it's not worth the effort to handle. - void *mem = (void *)&early_alloc_buf[allocated_bytes]; - allocated_bytes += size; - CHECK_LT(allocated_bytes, early_alloc_buf_size); - return mem; -} - -INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (EsanDuringInit && REAL(calloc) == nullptr) - return handleEarlyAlloc(size * n); - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, calloc, size, n); - void *res = REAL(calloc)(size, n); - // The memory is zeroed and thus is all written. - COMMON_INTERCEPTOR_WRITE_RANGE(nullptr, (uptr)res, size * n); - return res; -} - -INTERCEPTOR(void*, malloc, uptr size) { - if (EsanDuringInit && REAL(malloc) == nullptr) - return handleEarlyAlloc(size); - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, malloc, size); - return REAL(malloc)(size); -} - -INTERCEPTOR(void, free, void *p) { - void *ctx; - // There are only a few early allocation requests, so we simply skip the free. - if (isInEarlyAllocBuf(p)) - return; - COMMON_INTERCEPTOR_ENTER(ctx, free, p); - REAL(free)(p); -} - -namespace __esan { - -void initializeInterceptors() { - InitializeCommonInterceptors(); - - INTERCEPT_FUNCTION(strcpy); // NOLINT - INTERCEPT_FUNCTION(strncpy); - - INTERCEPT_FUNCTION(open); - ESAN_MAYBE_INTERCEPT_OPEN64; - INTERCEPT_FUNCTION(creat); - ESAN_MAYBE_INTERCEPT_CREAT64; - INTERCEPT_FUNCTION(unlink); - INTERCEPT_FUNCTION(rmdir); - - ESAN_MAYBE_INTERCEPT_SIGNAL; - ESAN_MAYBE_INTERCEPT_SIGACTION; - ESAN_MAYBE_INTERCEPT_SIGPROCMASK; - ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK; - - INTERCEPT_FUNCTION(calloc); - INTERCEPT_FUNCTION(malloc); - INTERCEPT_FUNCTION(free); - - // TODO(bruening): intercept routines that other sanitizers intercept that - // are not in the common pool or here yet, ideally by adding to the common - // pool. Examples include wcslen and bcopy. - - // TODO(bruening): there are many more libc routines that read or write data - // structures that no sanitizer is intercepting: sigaction, strtol, etc. -} - -} // namespace __esan diff --git a/lib/esan/esan_interface.cpp b/lib/esan/esan_interface.cpp deleted file mode 100644 index 43b3dff86f77..000000000000 --- a/lib/esan/esan_interface.cpp +++ /dev/null @@ -1,122 +0,0 @@ -//===-- esan_interface.cpp ------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -//===----------------------------------------------------------------------===// - -#include "esan_interface_internal.h" -#include "esan.h" -#include "sanitizer_common/sanitizer_internal_defs.h" - -using namespace __esan; // NOLINT - -void __esan_init(ToolType Tool, void *Ptr) { - if (Tool != __esan_which_tool) { - Printf("ERROR: tool mismatch: %d vs %d\n", Tool, __esan_which_tool); - Die(); - } - initializeLibrary(Tool); - processCompilationUnitInit(Ptr); -} - -void __esan_exit(void *Ptr) { - processCompilationUnitExit(Ptr); -} - -void __esan_aligned_load1(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 1, false); -} - -void __esan_aligned_load2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, false); -} - -void __esan_aligned_load4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, false); -} - -void __esan_aligned_load8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, false); -} - -void __esan_aligned_load16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, false); -} - -void __esan_aligned_store1(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 1, true); -} - -void __esan_aligned_store2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, true); -} - -void __esan_aligned_store4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, true); -} - -void __esan_aligned_store8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, true); -} - -void __esan_aligned_store16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, true); -} - -void __esan_unaligned_load2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, false); -} - -void __esan_unaligned_load4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, false); -} - -void __esan_unaligned_load8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, false); -} - -void __esan_unaligned_load16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, false); -} - -void __esan_unaligned_store2(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 2, true); -} - -void __esan_unaligned_store4(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 4, true); -} - -void __esan_unaligned_store8(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 8, true); -} - -void __esan_unaligned_store16(void *Addr) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, 16, true); -} - -void __esan_unaligned_loadN(void *Addr, uptr Size) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, Size, false); -} - -void __esan_unaligned_storeN(void *Addr, uptr Size) { - processRangeAccess(GET_CALLER_PC(), (uptr)Addr, Size, true); -} - -// Public interface: -extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE void __esan_report() { - reportResults(); -} - -SANITIZER_INTERFACE_ATTRIBUTE unsigned int __esan_get_sample_count() { - return getSampleCount(); -} -} // extern "C" diff --git a/lib/esan/esan_interface_internal.h b/lib/esan/esan_interface_internal.h deleted file mode 100644 index df51aa609aff..000000000000 --- a/lib/esan/esan_interface_internal.h +++ /dev/null @@ -1,83 +0,0 @@ -//===-- esan_interface_internal.h -------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Calls to the functions declared in this header will be inserted by -// the instrumentation module. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_INTERFACE_INTERNAL_H -#define ESAN_INTERFACE_INTERNAL_H - -#include <sanitizer_common/sanitizer_internal_defs.h> - -// This header should NOT include any other headers. -// All functions in this header are extern "C" and start with __esan_. - -using __sanitizer::uptr; -using __sanitizer::u32; - -extern "C" { - -// This should be kept consistent with LLVM's EfficiencySanitizerOptions. -// The value is passed as a 32-bit integer by the compiler. -typedef enum Type : u32 { - ESAN_None = 0, - ESAN_CacheFrag, - ESAN_WorkingSet, - ESAN_Max, -} ToolType; - -// To handle interceptors that invoke instrumented code prior to -// __esan_init() being called, the instrumentation module creates this -// global variable specifying the tool. -extern ToolType __esan_which_tool; - -// This function should be called at the very beginning of the process, -// before any instrumented code is executed and before any call to malloc. -SANITIZER_INTERFACE_ATTRIBUTE void __esan_init(ToolType Tool, void *Ptr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_exit(void *Ptr); - -// The instrumentation module will insert a call to one of these routines prior -// to each load and store instruction for which we do not have "fastpath" -// inlined instrumentation. These calls constitute the "slowpath" for our -// tools. We have separate routines for each type of memory access to enable -// targeted optimization. -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load1(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_load16(void *Addr); - -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store1(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_aligned_store16(void *Addr); - -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_load16(void *Addr); - -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store2(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store4(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store8(void *Addr); -SANITIZER_INTERFACE_ATTRIBUTE void __esan_unaligned_store16(void *Addr); - -// These cover unusually-sized accesses. -SANITIZER_INTERFACE_ATTRIBUTE -void __esan_unaligned_loadN(void *Addr, uptr Size); -SANITIZER_INTERFACE_ATTRIBUTE -void __esan_unaligned_storeN(void *Addr, uptr Size); - -} // extern "C" - -#endif // ESAN_INTERFACE_INTERNAL_H diff --git a/lib/esan/esan_linux.cpp b/lib/esan/esan_linux.cpp deleted file mode 100644 index 014205ce01eb..000000000000 --- a/lib/esan/esan_linux.cpp +++ /dev/null @@ -1,83 +0,0 @@ -//===-- esan.cpp ----------------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Linux-specific code for the Esan run-time. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX - -#include "esan.h" -#include "esan_shadow.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" -#include <sys/mman.h> -#include <errno.h> - -namespace __esan { - -void verifyAddressSpace() { -#if SANITIZER_LINUX && (defined(__x86_64__) || SANITIZER_MIPS64) - // The kernel determines its mmap base from the stack size limit. - // Our Linux 64-bit shadow mapping assumes the stack limit is less than a - // terabyte, which keeps the mmap region above 0x7e00'. - uptr StackLimit = GetStackSizeLimitInBytes(); - if (StackSizeIsUnlimited() || StackLimit > MaxStackSize) { - VReport(1, "The stack size limit is beyond the maximum supported.\n" - "Re-execing with a stack size below 1TB.\n"); - SetStackSizeLimitInBytes(MaxStackSize); - ReExec(); - } -#endif -} - -static bool liesWithinSingleAppRegion(uptr Start, SIZE_T Size) { - uptr AppStart, AppEnd; - for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) { - if (Start >= AppStart && Start + Size - 1 <= AppEnd) { - return true; - } - } - return false; -} - -bool fixMmapAddr(void **Addr, SIZE_T Size, int Flags) { - if (*Addr) { - if (!liesWithinSingleAppRegion((uptr)*Addr, Size)) { - VPrintf(1, "mmap conflict: [%p-%p) is not in an app region\n", - *Addr, (uptr)*Addr + Size); - if (Flags & MAP_FIXED) { - errno = EINVAL; - return false; - } else { - *Addr = 0; - } - } - } - return true; -} - -uptr checkMmapResult(uptr Addr, SIZE_T Size) { - if ((void *)Addr == MAP_FAILED) - return Addr; - if (!liesWithinSingleAppRegion(Addr, Size)) { - // FIXME: attempt to dynamically add this as an app region if it - // fits our shadow criteria. - // We could also try to remap somewhere else. - Printf("ERROR: unsupported mapping at [%p-%p)\n", Addr, Addr+Size); - Die(); - } - return Addr; -} - -} // namespace __esan - -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/esan/esan_shadow.h b/lib/esan/esan_shadow.h deleted file mode 100644 index b76a9ccbd9e4..000000000000 --- a/lib/esan/esan_shadow.h +++ /dev/null @@ -1,292 +0,0 @@ -//===-- esan_shadow.h -------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Shadow memory mappings for the esan run-time. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_SHADOW_H -#define ESAN_SHADOW_H - -#include "esan.h" -#include <sanitizer_common/sanitizer_platform.h> - -#if SANITIZER_WORDSIZE != 64 -#error Only 64-bit is supported -#endif - -namespace __esan { - -struct ApplicationRegion { - uptr Start; - uptr End; - bool ShadowMergedWithPrev; -}; - -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && defined(__x86_64__) -// Linux x86_64 -// -// Application memory falls into these 5 regions (ignoring the corner case -// of PIE with a non-zero PT_LOAD base): -// -// [0x00000000'00000000, 0x00000100'00000000) non-PIE + heap -// [0x00005500'00000000, 0x00005700'00000000) PIE -// [0x00007e00'00000000, 0x00007fff'ff600000) libraries + stack, part 1 -// [0x00007fff'ff601000, 0x00008000'00000000) libraries + stack, part 2 -// [0xffffffff'ff600000, 0xffffffff'ff601000) vsyscall -// -// Although we can ignore the vsyscall for the most part as there are few data -// references there (other sanitizers ignore it), we enforce a gap inside the -// library region to distinguish the vsyscall's shadow, considering this gap to -// be an invalid app region. -// We disallow application memory outside of those 5 regions. -// Our regions assume that the stack rlimit is less than a terabyte (otherwise -// the Linux kernel's default mmap region drops below 0x7e00'), which we enforce -// at init time (we can support larger and unlimited sizes for shadow -// scaledowns, but it is difficult for 1:1 mappings). -// -// Our shadow memory is scaled from a 1:1 mapping and supports a scale -// specified at library initialization time that can be any power-of-2 -// scaledown (1x, 2x, 4x, 8x, 16x, etc.). -// -// We model our shadow memory after Umbra, a library used by the Dr. Memory -// tool: https://github.com/DynamoRIO/drmemory/blob/master/umbra/umbra_x64.c. -// We use Umbra's scheme as it was designed to support different -// offsets, it supports two different shadow mappings (which we may want to -// use for future tools), and it ensures that the shadow of a shadow will -// not overlap either shadow memory or application memory. -// -// This formula translates from application memory to shadow memory: -// -// shadow(app) = ((app & 0x00000fff'ffffffff) + offset) >> scale -// -// Where the offset for 1:1 is 0x00001300'00000000. For other scales, the -// offset is shifted left by the scale, except for scales of 1 and 2 where -// it must be tweaked in order to pass the double-shadow test -// (see the "shadow(shadow)" comments below): -// scale == 0: 0x00001300'000000000 -// scale == 1: 0x00002200'000000000 -// scale == 2: 0x00004400'000000000 -// scale >= 3: (0x00001300'000000000 << scale) -// -// Do not pass in the open-ended end value to the formula as it will fail. -// -// The resulting shadow memory regions for a 0 scaling are: -// -// [0x00001300'00000000, 0x00001400'00000000) -// [0x00001800'00000000, 0x00001a00'00000000) -// [0x00002100'00000000, 0x000022ff'ff600000) -// [0x000022ff'ff601000, 0x00002300'00000000) -// [0x000022ff'ff600000, 0x000022ff'ff601000] -// -// We also want to ensure that a wild access by the application into the shadow -// regions will not corrupt our own shadow memory. shadow(shadow) ends up -// disjoint from shadow(app): -// -// [0x00001600'00000000, 0x00001700'00000000) -// [0x00001b00'00000000, 0x00001d00'00000000) -// [0x00001400'00000000, 0x000015ff'ff600000] -// [0x000015ff'ff601000, 0x00001600'00000000] -// [0x000015ff'ff600000, 0x000015ff'ff601000] - -static const struct ApplicationRegion AppRegions[] = { - {0x0000000000000000ull, 0x0000010000000000u, false}, - {0x0000550000000000u, 0x0000570000000000u, false}, - // We make one shadow mapping to hold the shadow regions for all 3 of these - // app regions, as the mappings interleave, and the gap between the 3rd and - // 4th scales down below a page. - {0x00007e0000000000u, 0x00007fffff600000u, false}, - {0x00007fffff601000u, 0x0000800000000000u, true}, - {0xffffffffff600000u, 0xffffffffff601000u, true}, -}; - -#elif SANITIZER_LINUX && SANITIZER_MIPS64 - -// Application memory falls into these 3 regions -// -// [0x00000001'00000000, 0x00000002'00000000) non-PIE + heap -// [0x000000aa'00000000, 0x000000ab'00000000) PIE -// [0x000000ff'00000000, 0x000000ff'ffffffff) libraries + stack -// -// This formula translates from application memory to shadow memory: -// -// shadow(app) = ((app & 0x00000f'ffffffff) + offset) >> scale -// -// Where the offset for 1:1 is 0x000013'00000000. For other scales, the -// offset is shifted left by the scale, except for scales of 1 and 2 where -// it must be tweaked in order to pass the double-shadow test -// (see the "shadow(shadow)" comments below): -// scale == 0: 0x000013'00000000 -// scale == 1: 0x000022'00000000 -// scale == 2: 0x000044'00000000 -// scale >= 3: (0x000013'00000000 << scale) -// -// The resulting shadow memory regions for a 0 scaling are: -// -// [0x00000014'00000000, 0x00000015'00000000) -// [0x0000001d'00000000, 0x0000001e'00000000) -// [0x00000022'00000000, 0x00000022'ffffffff) -// -// We also want to ensure that a wild access by the application into the shadow -// regions will not corrupt our own shadow memory. shadow(shadow) ends up -// disjoint from shadow(app): -// -// [0x00000017'00000000, 0x00000018'00000000) -// [0x00000020'00000000, 0x00000021'00000000) -// [0x00000015'00000000, 0x00000015'ffffffff] - -static const struct ApplicationRegion AppRegions[] = { - {0x0100000000u, 0x0200000000u, false}, - {0xaa00000000u, 0xab00000000u, false}, - {0xff00000000u, 0xffffffffffu, false}, -}; - -#else -#error Platform not supported -#endif - -static const u32 NumAppRegions = sizeof(AppRegions)/sizeof(AppRegions[0]); - -// See the comment above: we do not currently support a stack size rlimit -// equal to or larger than 1TB. -static const uptr MaxStackSize = (1ULL << 40) - 4096; - -class ShadowMapping { -public: - - // The scale and offset vary by tool. - uptr Scale; - uptr Offset; - - // TODO(sagar.thakur): Try to hardcode the mask as done in the compiler - // instrumentation to reduce the runtime cost of appToShadow. - struct ShadowMemoryMask40 { - static const uptr Mask = 0x0000000fffffffffu; - }; - - struct ShadowMemoryMask47 { - static const uptr Mask = 0x00000fffffffffffu; - }; - - void initialize(uptr ShadowScale) { - - const uptr OffsetArray40[3] = { - 0x0000001300000000u, - 0x0000002200000000u, - 0x0000004400000000u, - }; - - const uptr OffsetArray47[3] = { - 0x0000130000000000u, - 0x0000220000000000u, - 0x0000440000000000u, - }; - - Scale = ShadowScale; - switch (VmaSize) { - case 40: { - if (Scale <= 2) - Offset = OffsetArray40[Scale]; - else - Offset = OffsetArray40[0] << Scale; - } - break; - case 47: { - if (Scale <= 2) - Offset = OffsetArray47[Scale]; - else - Offset = OffsetArray47[0] << Scale; - } - break; - default: { - Printf("ERROR: %d-bit virtual memory address size not supported\n", VmaSize); - Die(); - } - } - } -}; -extern ShadowMapping Mapping; - -static inline bool getAppRegion(u32 i, uptr *Start, uptr *End) { - if (i >= NumAppRegions) - return false; - *Start = AppRegions[i].Start; - *End = AppRegions[i].End; - return true; -} - -ALWAYS_INLINE -bool isAppMem(uptr Mem) { - for (u32 i = 0; i < NumAppRegions; ++i) { - if (Mem >= AppRegions[i].Start && Mem < AppRegions[i].End) - return true; - } - return false; -} - -template<typename Params> -uptr appToShadowImpl(uptr App) { - return (((App & Params::Mask) + Mapping.Offset) >> Mapping.Scale); -} - -ALWAYS_INLINE -uptr appToShadow(uptr App) { - switch (VmaSize) { - case 40: return appToShadowImpl<ShadowMapping::ShadowMemoryMask40>(App); - case 47: return appToShadowImpl<ShadowMapping::ShadowMemoryMask47>(App); - default: { - Printf("ERROR: %d-bit virtual memory address size not supported\n", VmaSize); - Die(); - } - } -} - -static inline bool getShadowRegion(u32 i, uptr *Start, uptr *End) { - if (i >= NumAppRegions) - return false; - u32 UnmergedShadowCount = 0; - u32 AppIdx; - for (AppIdx = 0; AppIdx < NumAppRegions; ++AppIdx) { - if (!AppRegions[AppIdx].ShadowMergedWithPrev) { - if (UnmergedShadowCount == i) - break; - UnmergedShadowCount++; - } - } - if (AppIdx >= NumAppRegions || UnmergedShadowCount != i) - return false; - *Start = appToShadow(AppRegions[AppIdx].Start); - // The formula fails for the end itself. - *End = appToShadow(AppRegions[AppIdx].End - 1) + 1; - // Merge with adjacent shadow regions: - for (++AppIdx; AppIdx < NumAppRegions; ++AppIdx) { - if (!AppRegions[AppIdx].ShadowMergedWithPrev) - break; - *Start = Min(*Start, appToShadow(AppRegions[AppIdx].Start)); - *End = Max(*End, appToShadow(AppRegions[AppIdx].End - 1) + 1); - } - return true; -} - -ALWAYS_INLINE -bool isShadowMem(uptr Mem) { - // We assume this is not used on any critical performance path and so there's - // no need to hardcode the mapping results. - for (uptr i = 0; i < NumAppRegions; ++i) { - if (Mem >= appToShadow(AppRegions[i].Start) && - Mem < appToShadow(AppRegions[i].End - 1) + 1) - return true; - } - return false; -} - -} // namespace __esan - -#endif /* ESAN_SHADOW_H */ diff --git a/lib/esan/esan_sideline.h b/lib/esan/esan_sideline.h deleted file mode 100644 index 74551fbde850..000000000000 --- a/lib/esan/esan_sideline.h +++ /dev/null @@ -1,64 +0,0 @@ -//===-- esan_sideline.h -----------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Esan sideline thread support. -//===----------------------------------------------------------------------===// - -#ifndef ESAN_SIDELINE_H -#define ESAN_SIDELINE_H - -#include "sanitizer_common/sanitizer_atomic.h" -#include "sanitizer_common/sanitizer_internal_defs.h" -#include "sanitizer_common/sanitizer_platform_limits_freebsd.h" -#include "sanitizer_common/sanitizer_platform_limits_posix.h" - -namespace __esan { - -typedef void (*SidelineFunc)(void *Arg); - -// Currently only one sideline thread is supported. -// It calls the SidelineFunc passed to launchThread once on each sample at the -// given frequency in real time (i.e., wall clock time). -class SidelineThread { -public: - // We cannot initialize any fields in the constructor as it will be called - // *after* launchThread for a static instance, as esan.module_ctor is called - // before static initializers. - SidelineThread() {} - ~SidelineThread() {} - - // To simplify declaration in sanitizer code where we want to avoid - // heap allocations, the constructor and destructor do nothing and - // launchThread and joinThread do the real work. - // They should each be called just once. - bool launchThread(SidelineFunc takeSample, void *Arg, u32 FreqMilliSec); - bool joinThread(); - - // Must be called from the sideline thread itself. - bool adjustTimer(u32 FreqMilliSec); - -private: - static int runSideline(void *Arg); - static void registerSignal(int SigNum); - static void handleSidelineSignal(int SigNum, __sanitizer_siginfo *SigInfo, - void *Ctx); - - char *Stack; - SidelineFunc sampleFunc; - void *FuncArg; - u32 Freq; - uptr SidelineId; - atomic_uintptr_t SidelineExit; -}; - -} // namespace __esan - -#endif // ESAN_SIDELINE_H diff --git a/lib/esan/esan_sideline_bsd.cpp b/lib/esan/esan_sideline_bsd.cpp deleted file mode 100644 index 3134d3776730..000000000000 --- a/lib/esan/esan_sideline_bsd.cpp +++ /dev/null @@ -1,35 +0,0 @@ -//===-- esan_sideline_bsd.cpp -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Support for a separate or "sideline" tool thread on FreeBSD. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD - -#include "esan_sideline.h" - -namespace __esan { - -static SidelineThread *TheThread; - -bool SidelineThread::launchThread(SidelineFunc takeSample, void *Arg, - u32 FreqMilliSec) { - return true; -} - -bool SidelineThread::joinThread() { - return true; -} - -} // namespace __esan - -#endif // SANITIZER_FREEBSD diff --git a/lib/esan/esan_sideline_linux.cpp b/lib/esan/esan_sideline_linux.cpp deleted file mode 100644 index 2de25fba700d..000000000000 --- a/lib/esan/esan_sideline_linux.cpp +++ /dev/null @@ -1,178 +0,0 @@ -//===-- esan_sideline_linux.cpp ---------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Support for a separate or "sideline" tool thread on Linux. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_LINUX - -#include "esan_sideline.h" -#include "sanitizer_common/sanitizer_atomic.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_linux.h" -#include <errno.h> -#include <sched.h> -#include <sys/prctl.h> -#include <sys/signal.h> -#include <sys/time.h> -#include <sys/types.h> -#include <sys/wait.h> - -namespace __esan { - -static const int SigAltStackSize = 4*1024; -static const int SidelineStackSize = 4*1024; -static const uptr SidelineIdUninitialized = 1; - -// FIXME: we'll need some kind of TLS (can we trust that a pthread key will -// work in our non-POSIX thread?) to access our data in our signal handler -// with multiple sideline threads. For now we assume there is only one -// sideline thread and we use a dirty solution of a global var. -static SidelineThread *TheThread; - -// We aren't passing SA_NODEFER so the same signal is blocked while here. -void SidelineThread::handleSidelineSignal(int SigNum, - __sanitizer_siginfo *SigInfo, - void *Ctx) { - VPrintf(3, "Sideline signal %d\n", SigNum); - CHECK_EQ(SigNum, SIGALRM); - // See above about needing TLS to avoid this global var. - SidelineThread *Thread = TheThread; - if (atomic_load(&Thread->SidelineExit, memory_order_relaxed) != 0) - return; - Thread->sampleFunc(Thread->FuncArg); -} - -void SidelineThread::registerSignal(int SigNum) { - __sanitizer_sigaction SigAct; - internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = handleSidelineSignal; - // We do not pass SA_NODEFER as we want to block the same signal. - SigAct.sa_flags = SA_ONSTACK | SA_SIGINFO; - int Res = internal_sigaction(SigNum, &SigAct, nullptr); - CHECK_EQ(Res, 0); -} - -int SidelineThread::runSideline(void *Arg) { - VPrintf(1, "Sideline thread starting\n"); - SidelineThread *Thread = static_cast<SidelineThread*>(Arg); - - // If the parent dies, we want to exit also. - internal_prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); - - // Set up a signal handler on an alternate stack for safety. - InternalMmapVector<char> StackMap(SigAltStackSize); - stack_t SigAltStack; - SigAltStack.ss_sp = StackMap.data(); - SigAltStack.ss_size = SigAltStackSize; - SigAltStack.ss_flags = 0; - internal_sigaltstack(&SigAltStack, nullptr); - - // We inherit the signal mask from the app thread. In case - // we weren't created at init time, we ensure the mask is empty. - __sanitizer_sigset_t SigSet; - internal_sigfillset(&SigSet); - int Res = internal_sigprocmask(SIG_UNBLOCK, &SigSet, nullptr); - CHECK_EQ(Res, 0); - - registerSignal(SIGALRM); - - bool TimerSuccess = Thread->adjustTimer(Thread->Freq); - CHECK(TimerSuccess); - - // We loop, doing nothing but handling itimer signals. - while (atomic_load(&TheThread->SidelineExit, memory_order_relaxed) == 0) - sched_yield(); - - if (!Thread->adjustTimer(0)) - VPrintf(1, "Failed to disable timer\n"); - - VPrintf(1, "Sideline thread exiting\n"); - return 0; -} - -bool SidelineThread::launchThread(SidelineFunc takeSample, void *Arg, - u32 FreqMilliSec) { - // This can only be called once. However, we can't clear a field in - // the constructor and check for that here as the constructor for - // a static instance is called *after* our module_ctor and thus after - // this routine! Thus we rely on the TheThread check below. - CHECK(TheThread == nullptr); // Only one sideline thread is supported. - TheThread = this; - sampleFunc = takeSample; - FuncArg = Arg; - Freq = FreqMilliSec; - atomic_store(&SidelineExit, 0, memory_order_relaxed); - - // We do without a guard page. - Stack = static_cast<char*>(MmapOrDie(SidelineStackSize, "SidelineStack")); - // We need to handle the return value from internal_clone() not having been - // assigned yet (for our CHECK in adjustTimer()) so we ensure this has a - // sentinel value. - SidelineId = SidelineIdUninitialized; - // By omitting CLONE_THREAD, the child is in its own thread group and will not - // receive any of the application's signals. - SidelineId = internal_clone( - runSideline, Stack + SidelineStackSize, - CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_UNTRACED, - this, nullptr /* parent_tidptr */, - nullptr /* newtls */, nullptr /* child_tidptr */); - int ErrCode; - if (internal_iserror(SidelineId, &ErrCode)) { - Printf("FATAL: EfficiencySanitizer failed to spawn a thread (code %d).\n", - ErrCode); - Die(); - return false; // Not reached. - } - return true; -} - -bool SidelineThread::joinThread() { - VPrintf(1, "Joining sideline thread\n"); - bool Res = true; - atomic_store(&SidelineExit, 1, memory_order_relaxed); - while (true) { - uptr Status = internal_waitpid(SidelineId, nullptr, __WALL); - int ErrCode; - if (!internal_iserror(Status, &ErrCode)) - break; - if (ErrCode == EINTR) - continue; - VPrintf(1, "Failed to join sideline thread (errno %d)\n", ErrCode); - Res = false; - break; - } - UnmapOrDie(Stack, SidelineStackSize); - return Res; -} - -// Must be called from the sideline thread itself. -bool SidelineThread::adjustTimer(u32 FreqMilliSec) { - // The return value of internal_clone() may not have been assigned yet: - CHECK(internal_getpid() == SidelineId || - SidelineId == SidelineIdUninitialized); - Freq = FreqMilliSec; - struct itimerval TimerVal; - TimerVal.it_interval.tv_sec = (time_t) Freq / 1000; - TimerVal.it_interval.tv_usec = (time_t) (Freq % 1000) * 1000; - TimerVal.it_value.tv_sec = (time_t) Freq / 1000; - TimerVal.it_value.tv_usec = (time_t) (Freq % 1000) * 1000; - // As we're in a different thread group, we cannot use either - // ITIMER_PROF or ITIMER_VIRTUAL without taking up scheduled - // time ourselves: thus we must use real time. - int Res = setitimer(ITIMER_REAL, &TimerVal, nullptr); - return (Res == 0); -} - -} // namespace __esan - -#endif // SANITIZER_LINUX diff --git a/lib/esan/working_set.cpp b/lib/esan/working_set.cpp deleted file mode 100644 index e56902c8f32a..000000000000 --- a/lib/esan/working_set.cpp +++ /dev/null @@ -1,280 +0,0 @@ -//===-- working_set.cpp ---------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// This file contains working-set-specific code. -//===----------------------------------------------------------------------===// - -#include "working_set.h" -#include "esan.h" -#include "esan_circular_buffer.h" -#include "esan_flags.h" -#include "esan_shadow.h" -#include "esan_sideline.h" -#include "sanitizer_common/sanitizer_procmaps.h" - -// We shadow every cache line of app memory with one shadow byte. -// - The highest bit of each shadow byte indicates whether the corresponding -// cache line has ever been accessed. -// - The lowest bit of each shadow byte indicates whether the corresponding -// cache line was accessed since the last sample. -// - The other bits are used for working set snapshots at successively -// lower frequencies, each bit to the left from the lowest bit stepping -// down the frequency by 2 to the power of getFlags()->snapshot_step. -// Thus we have something like this: -// Bit 0: Since last sample -// Bit 1: Since last 2^2 samples -// Bit 2: Since last 2^4 samples -// Bit 3: ... -// Bit 7: Ever accessed. -// We live with races in accessing each shadow byte. -typedef unsigned char byte; - -namespace __esan { - -// Our shadow memory assumes that the line size is 64. -static const u32 CacheLineSize = 64; - -// See the shadow byte layout description above. -static const u32 TotalWorkingSetBitIdx = 7; -// We accumulate to the left until we hit this bit. -// We don't need to accumulate to the final bit as it's set on each ref -// by the compiler instrumentation. -static const u32 MaxAccumBitIdx = 6; -static const u32 CurWorkingSetBitIdx = 0; -static const byte ShadowAccessedVal = - (1 << TotalWorkingSetBitIdx) | (1 << CurWorkingSetBitIdx); - -static SidelineThread Thread; -// If we use real-time-based timer samples this won't overflow in any realistic -// scenario, but if we switch to some other unit (such as memory accesses) we -// may want to consider a 64-bit int. -static u32 SnapshotNum; - -// We store the wset size for each of 8 different sampling frequencies. -static const u32 NumFreq = 8; // One for each bit of our shadow bytes. -// We cannot use static objects as the global destructor is called -// prior to our finalize routine. -// These are each circular buffers, sized up front. -CircularBuffer<u32> SizePerFreq[NumFreq]; -// We cannot rely on static initializers (they may run too late) but -// we record the size here for clarity: -u32 CircularBufferSizes[NumFreq] = { - // These are each mmap-ed so our minimum is one page. - 32*1024, - 16*1024, - 8*1024, - 4*1024, - 4*1024, - 4*1024, - 4*1024, - 4*1024, -}; - -void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size, - bool IsWrite) { - if (Size == 0) - return; - SIZE_T I = 0; - uptr LineSize = getFlags()->cache_line_size; - // As Addr+Size could overflow at the top of a 32-bit address space, - // we avoid the simpler formula that rounds the start and end. - SIZE_T NumLines = Size / LineSize + - // Add any extra at the start or end adding on an extra line: - (LineSize - 1 + Addr % LineSize + Size % LineSize) / LineSize; - byte *Shadow = (byte *)appToShadow(Addr); - // Write shadow bytes until we're word-aligned. - while (I < NumLines && (uptr)Shadow % 4 != 0) { - if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal) - *Shadow |= ShadowAccessedVal; - ++Shadow; - ++I; - } - // Write whole shadow words at a time. - // Using a word-stride loop improves the runtime of a microbenchmark of - // memset calls by 10%. - u32 WordValue = ShadowAccessedVal | ShadowAccessedVal << 8 | - ShadowAccessedVal << 16 | ShadowAccessedVal << 24; - while (I + 4 <= NumLines) { - if ((*(u32*)Shadow & WordValue) != WordValue) - *(u32*)Shadow |= WordValue; - Shadow += 4; - I += 4; - } - // Write any trailing shadow bytes. - while (I < NumLines) { - if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal) - *Shadow |= ShadowAccessedVal; - ++Shadow; - ++I; - } -} - -// This routine will word-align ShadowStart and ShadowEnd prior to scanning. -// It does *not* clear for BitIdx==TotalWorkingSetBitIdx, as that top bit -// measures the access during the entire execution and should never be cleared. -static u32 countAndClearShadowValues(u32 BitIdx, uptr ShadowStart, - uptr ShadowEnd) { - u32 WorkingSetSize = 0; - u32 ByteValue = 0x1 << BitIdx; - u32 WordValue = ByteValue | ByteValue << 8 | ByteValue << 16 | - ByteValue << 24; - // Get word aligned start. - ShadowStart = RoundDownTo(ShadowStart, sizeof(u32)); - bool Accum = getFlags()->record_snapshots && BitIdx < MaxAccumBitIdx; - // Do not clear the bit that measures access during the entire execution. - bool Clear = BitIdx < TotalWorkingSetBitIdx; - for (u32 *Ptr = (u32 *)ShadowStart; Ptr < (u32 *)ShadowEnd; ++Ptr) { - if ((*Ptr & WordValue) != 0) { - byte *BytePtr = (byte *)Ptr; - for (u32 j = 0; j < sizeof(u32); ++j) { - if (BytePtr[j] & ByteValue) { - ++WorkingSetSize; - if (Accum) { - // Accumulate to the lower-frequency bit to the left. - BytePtr[j] |= (ByteValue << 1); - } - } - } - if (Clear) { - // Clear this bit from every shadow byte. - *Ptr &= ~WordValue; - } - } - } - return WorkingSetSize; -} - -// Scan shadow memory to calculate the number of cache lines being accessed, -// i.e., the number of non-zero bits indexed by BitIdx in each shadow byte. -// We also clear the lowest bits (most recent working set snapshot). -// We do *not* clear for BitIdx==TotalWorkingSetBitIdx, as that top bit -// measures the access during the entire execution and should never be cleared. -static u32 computeWorkingSizeAndReset(u32 BitIdx) { - u32 WorkingSetSize = 0; - MemoryMappingLayout MemIter(true/*cache*/); - MemoryMappedSegment Segment; - while (MemIter.Next(&Segment)) { - VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n", __FUNCTION__, - Segment.start, Segment.end, Segment.protection, - isAppMem(Segment.start), isShadowMem(Segment.start)); - if (isShadowMem(Segment.start) && Segment.IsWritable()) { - VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Segment.start, - Segment.end); - WorkingSetSize += - countAndClearShadowValues(BitIdx, Segment.start, Segment.end); - } - } - return WorkingSetSize; -} - -// This is invoked from a signal handler but in a sideline thread doing nothing -// else so it is a little less fragile than a typical signal handler. -static void takeSample(void *Arg) { - u32 BitIdx = CurWorkingSetBitIdx; - u32 Freq = 1; - ++SnapshotNum; // Simpler to skip 0 whose mod matches everything. - while (BitIdx <= MaxAccumBitIdx && (SnapshotNum % Freq) == 0) { - u32 NumLines = computeWorkingSizeAndReset(BitIdx); - VReport(1, "%s: snapshot #%5d bit %d freq %4d: %8u\n", SanitizerToolName, - SnapshotNum, BitIdx, Freq, NumLines); - SizePerFreq[BitIdx].push_back(NumLines); - Freq = Freq << getFlags()->snapshot_step; - BitIdx++; - } -} - -unsigned int getSampleCountWorkingSet() -{ - return SnapshotNum; -} - -// Initialization that must be done before any instrumented code is executed. -void initializeShadowWorkingSet() { - CHECK(getFlags()->cache_line_size == CacheLineSize); - registerMemoryFaultHandler(); -} - -void initializeWorkingSet() { - if (getFlags()->record_snapshots) { - for (u32 i = 0; i < NumFreq; ++i) - SizePerFreq[i].initialize(CircularBufferSizes[i]); - Thread.launchThread(takeSample, nullptr, getFlags()->sample_freq); - } -} - -static u32 getPeriodForPrinting(u32 MilliSec, const char *&Unit) { - if (MilliSec > 600000) { - Unit = "min"; - return MilliSec / 60000; - } else if (MilliSec > 10000) { - Unit = "sec"; - return MilliSec / 1000; - } else { - Unit = "ms"; - return MilliSec; - } -} - -static u32 getSizeForPrinting(u32 NumOfCachelines, const char *&Unit) { - // We need a constant to avoid software divide support: - static const u32 KilobyteCachelines = (0x1 << 10) / CacheLineSize; - static const u32 MegabyteCachelines = KilobyteCachelines << 10; - - if (NumOfCachelines > 10 * MegabyteCachelines) { - Unit = "MB"; - return NumOfCachelines / MegabyteCachelines; - } else if (NumOfCachelines > 10 * KilobyteCachelines) { - Unit = "KB"; - return NumOfCachelines / KilobyteCachelines; - } else { - Unit = "Bytes"; - return NumOfCachelines * CacheLineSize; - } -} - -void reportWorkingSet() { - const char *Unit; - if (getFlags()->record_snapshots) { - u32 Freq = 1; - Report(" Total number of samples: %u\n", SnapshotNum); - for (u32 i = 0; i < NumFreq; ++i) { - u32 Time = getPeriodForPrinting(getFlags()->sample_freq*Freq, Unit); - Report(" Samples array #%d at period %u %s\n", i, Time, Unit); - // FIXME: report whether we wrapped around and thus whether we - // have data on the whole run or just the last N samples. - for (u32 j = 0; j < SizePerFreq[i].size(); ++j) { - u32 Size = getSizeForPrinting(SizePerFreq[i][j], Unit); - Report("#%4d: %8u %s (%9u cache lines)\n", j, Size, Unit, - SizePerFreq[i][j]); - } - Freq = Freq << getFlags()->snapshot_step; - } - } - - // Get the working set size for the entire execution. - u32 NumOfCachelines = computeWorkingSizeAndReset(TotalWorkingSetBitIdx); - u32 Size = getSizeForPrinting(NumOfCachelines, Unit); - Report(" %s: the total working set size: %u %s (%u cache lines)\n", - SanitizerToolName, Size, Unit, NumOfCachelines); -} - -int finalizeWorkingSet() { - if (getFlags()->record_snapshots) - Thread.joinThread(); - reportWorkingSet(); - if (getFlags()->record_snapshots) { - for (u32 i = 0; i < NumFreq; ++i) - SizePerFreq[i].free(); - } - return 0; -} - -} // namespace __esan diff --git a/lib/esan/working_set.h b/lib/esan/working_set.h deleted file mode 100644 index 6a976c3f9b22..000000000000 --- a/lib/esan/working_set.h +++ /dev/null @@ -1,40 +0,0 @@ -//===-- working_set.h -------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// Header for working-set-specific code. -//===----------------------------------------------------------------------===// - -#ifndef WORKING_SET_H -#define WORKING_SET_H - -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_internal_defs.h" - -namespace __esan { - -void initializeWorkingSet(); -void initializeShadowWorkingSet(); -int finalizeWorkingSet(); -void reportWorkingSet(); -unsigned int getSampleCountWorkingSet(); -void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size, - bool IsWrite); - -// Platform-dependent. -void registerMemoryFaultHandler(); -bool processWorkingSetSignal(int SigNum, void (*Handler)(int), - void (**Result)(int)); -bool processWorkingSetSigaction(int SigNum, const void *Act, void *OldAct); -bool processWorkingSetSigprocmask(int How, void *Set, void *OldSet); - -} // namespace __esan - -#endif // WORKING_SET_H diff --git a/lib/esan/working_set_posix.cpp b/lib/esan/working_set_posix.cpp deleted file mode 100644 index 5ec53b959261..000000000000 --- a/lib/esan/working_set_posix.cpp +++ /dev/null @@ -1,134 +0,0 @@ -//===-- working_set_posix.cpp -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners. -// -// POSIX-specific working set tool code. -//===----------------------------------------------------------------------===// - -#include "working_set.h" -#include "esan_flags.h" -#include "esan_shadow.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_linux.h" -#include <signal.h> -#include <sys/mman.h> - -namespace __esan { - -// We only support regular POSIX threads with a single signal handler -// for the whole process == thread group. -// Thus we only need to store one app signal handler. -// FIXME: Store and use any alternate stack and signal flags set by -// the app. For now we just call the app handler from our handler. -static __sanitizer_sigaction AppSigAct; - -bool processWorkingSetSignal(int SigNum, void (*Handler)(int), - void (**Result)(int)) { - VPrintf(2, "%s: %d\n", __FUNCTION__, SigNum); - if (SigNum == SIGSEGV) { - *Result = AppSigAct.handler; - AppSigAct.sigaction = (decltype(AppSigAct.sigaction))Handler; - return false; // Skip real call. - } - return true; -} - -bool processWorkingSetSigaction(int SigNum, const void *ActVoid, - void *OldActVoid) { - VPrintf(2, "%s: %d\n", __FUNCTION__, SigNum); - if (SigNum == SIGSEGV) { - const struct sigaction *Act = (const struct sigaction *) ActVoid; - struct sigaction *OldAct = (struct sigaction *) OldActVoid; - if (OldAct) - internal_memcpy(OldAct, &AppSigAct, sizeof(OldAct)); - if (Act) - internal_memcpy(&AppSigAct, Act, sizeof(AppSigAct)); - return false; // Skip real call. - } - return true; -} - -bool processWorkingSetSigprocmask(int How, void *Set, void *OldSet) { - VPrintf(2, "%s\n", __FUNCTION__); - // All we need to do is ensure that SIGSEGV is not blocked. - // FIXME: we are not fully transparent as we do not pretend that - // SIGSEGV is still blocked on app queries: that would require - // per-thread mask tracking. - if (Set && (How == SIG_BLOCK || How == SIG_SETMASK)) { - if (internal_sigismember((__sanitizer_sigset_t *)Set, SIGSEGV)) { - VPrintf(1, "%s: removing SIGSEGV from the blocked set\n", __FUNCTION__); - internal_sigdelset((__sanitizer_sigset_t *)Set, SIGSEGV); - } - } - return true; -} - -static void reinstateDefaultHandler(int SigNum) { - __sanitizer_sigaction SigAct; - internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = (decltype(SigAct.sigaction))SIG_DFL; - int Res = internal_sigaction(SigNum, &SigAct, nullptr); - CHECK(Res == 0); - VPrintf(1, "Unregistered for %d handler\n", SigNum); -} - -// If this is a shadow fault, we handle it here; otherwise, we pass it to the -// app to handle it just as the app would do without our tool in place. -static void handleMemoryFault(int SigNum, __sanitizer_siginfo *Info, - void *Ctx) { - if (SigNum == SIGSEGV) { - // We rely on si_addr being filled in (thus we do not support old kernels). - siginfo_t *SigInfo = (siginfo_t *)Info; - uptr Addr = (uptr)SigInfo->si_addr; - if (isShadowMem(Addr)) { - VPrintf(3, "Shadow fault @%p\n", Addr); - uptr PageSize = GetPageSizeCached(); - int Res = internal_mprotect((void *)RoundDownTo(Addr, PageSize), - PageSize, PROT_READ|PROT_WRITE); - CHECK(Res == 0); - } else if (AppSigAct.sigaction) { - // FIXME: For simplicity we ignore app options including its signal stack - // (we just use ours) and all the delivery flags. - AppSigAct.sigaction(SigNum, Info, Ctx); - } else { - // Crash instead of spinning with infinite faults. - reinstateDefaultHandler(SigNum); - } - } else - UNREACHABLE("signal not registered"); -} - -void registerMemoryFaultHandler() { - // We do not use an alternate signal stack, as doing so would require - // setting it up for each app thread. - // FIXME: This could result in problems with emulating the app's signal - // handling if the app relies on an alternate stack for SIGSEGV. - - // We require that SIGSEGV is not blocked. We use a sigprocmask - // interceptor to ensure that in the future. Here we ensure it for - // the current thread. We assume there are no other threads at this - // point during initialization, or that at least they do not block - // SIGSEGV. - __sanitizer_sigset_t SigSet; - internal_sigemptyset(&SigSet); - internal_sigprocmask(SIG_BLOCK, &SigSet, nullptr); - - __sanitizer_sigaction SigAct; - internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = handleMemoryFault; - // We want to handle nested signals b/c we need to handle a - // shadow fault in an app signal handler. - SigAct.sa_flags = SA_SIGINFO | SA_NODEFER; - int Res = internal_sigaction(SIGSEGV, &SigAct, &AppSigAct); - CHECK(Res == 0); - VPrintf(1, "Registered for SIGSEGV handler\n"); -} - -} // namespace __esan diff --git a/lib/fuzzer/FuzzerBuiltins.h b/lib/fuzzer/FuzzerBuiltins.h index a80938d9a54d..5f1ccef8a9ca 100644 --- a/lib/fuzzer/FuzzerBuiltins.h +++ b/lib/fuzzer/FuzzerBuiltins.h @@ -1,9 +1,8 @@ //===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Wrapper functions and marcos around builtin functions. diff --git a/lib/fuzzer/FuzzerBuiltinsMsvc.h b/lib/fuzzer/FuzzerBuiltinsMsvc.h index 67dd57ff9a93..82709cfe7b40 100644 --- a/lib/fuzzer/FuzzerBuiltinsMsvc.h +++ b/lib/fuzzer/FuzzerBuiltinsMsvc.h @@ -1,9 +1,8 @@ //===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Wrapper functions and marcos that use intrinsics instead of builtin functions @@ -25,7 +24,7 @@ // __builtin_return_address() cannot be compiled with MSVC. Use the equivalent // from <intrin.h> -#define GET_CALLER_PC() reinterpret_cast<uintptr_t>(_ReturnAddress()) +#define GET_CALLER_PC() _ReturnAddress() namespace fuzzer { diff --git a/lib/fuzzer/FuzzerCommand.h b/lib/fuzzer/FuzzerCommand.h index 9d258a228f89..87308864af53 100644 --- a/lib/fuzzer/FuzzerCommand.h +++ b/lib/fuzzer/FuzzerCommand.h @@ -1,9 +1,8 @@ //===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // FuzzerCommand represents a command to run in a subprocess. It allows callers diff --git a/lib/fuzzer/FuzzerCorpus.h b/lib/fuzzer/FuzzerCorpus.h index f844c07c7572..6a95ef3a8e64 100644 --- a/lib/fuzzer/FuzzerCorpus.h +++ b/lib/fuzzer/FuzzerCorpus.h @@ -1,9 +1,8 @@ //===- FuzzerCorpus.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::InputCorpus @@ -86,9 +85,10 @@ class InputCorpus { bool empty() const { return Inputs.empty(); } const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } - void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, - bool HasFocusFunction, const Vector<uint32_t> &FeatureSet, - const DataFlowTrace &DFT, const InputInfo *BaseII) { + InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, + bool HasFocusFunction, + const Vector<uint32_t> &FeatureSet, + const DataFlowTrace &DFT, const InputInfo *BaseII) { assert(!U.empty()); if (FeatureDebug) Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); @@ -114,6 +114,7 @@ class InputCorpus { UpdateCorpusDistribution(); PrintCorpus(); // ValidateFeatureSet(); + return &II; } // Debug-only @@ -170,7 +171,7 @@ class InputCorpus { InputInfo &II = *Inputs[ChooseUnitIdxToMutate(Rand)]; assert(!II.U.empty()); return II; - }; + } // Returns an index of random unit from the corpus to mutate. size_t ChooseUnitIdxToMutate(Random &Rand) { diff --git a/lib/fuzzer/FuzzerCrossOver.cpp b/lib/fuzzer/FuzzerCrossOver.cpp index 8b0fd7d529a8..83d9f8d47cb1 100644 --- a/lib/fuzzer/FuzzerCrossOver.cpp +++ b/lib/fuzzer/FuzzerCrossOver.cpp @@ -1,9 +1,8 @@ //===- FuzzerCrossOver.cpp - Cross over two test inputs -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Cross over test inputs. diff --git a/lib/fuzzer/FuzzerDataFlowTrace.cpp b/lib/fuzzer/FuzzerDataFlowTrace.cpp index 764f3e49fd2d..99ff918f7c6c 100644 --- a/lib/fuzzer/FuzzerDataFlowTrace.cpp +++ b/lib/fuzzer/FuzzerDataFlowTrace.cpp @@ -1,91 +1,281 @@ //===- FuzzerDataFlowTrace.cpp - DataFlowTrace ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::DataFlowTrace //===----------------------------------------------------------------------===// #include "FuzzerDataFlowTrace.h" + +#include "FuzzerCommand.h" #include "FuzzerIO.h" +#include "FuzzerRandom.h" +#include "FuzzerSHA1.h" +#include "FuzzerUtil.h" #include <cstdlib> #include <fstream> +#include <numeric> +#include <queue> +#include <sstream> #include <string> +#include <unordered_map> +#include <unordered_set> #include <vector> namespace fuzzer { +static const char *kFunctionsTxt = "functions.txt"; + +bool BlockCoverage::AppendCoverage(const std::string &S) { + std::stringstream SS(S); + return AppendCoverage(SS); +} + +// Coverage lines have this form: +// CN X Y Z T +// where N is the number of the function, T is the total number of instrumented +// BBs, and X,Y,Z, if present, are the indecies of covered BB. +// BB #0, which is the entry block, is not explicitly listed. +bool BlockCoverage::AppendCoverage(std::istream &IN) { + std::string L; + while (std::getline(IN, L, '\n')) { + if (L.empty()) + continue; + std::stringstream SS(L.c_str() + 1); + size_t FunctionId = 0; + SS >> FunctionId; + if (L[0] == 'F') { + FunctionsWithDFT.insert(FunctionId); + continue; + } + if (L[0] != 'C') continue; + Vector<uint32_t> CoveredBlocks; + while (true) { + uint32_t BB = 0; + SS >> BB; + if (!SS) break; + CoveredBlocks.push_back(BB); + } + if (CoveredBlocks.empty()) return false; + uint32_t NumBlocks = CoveredBlocks.back(); + CoveredBlocks.pop_back(); + for (auto BB : CoveredBlocks) + if (BB >= NumBlocks) return false; + auto It = Functions.find(FunctionId); + auto &Counters = + It == Functions.end() + ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)}) + .first->second + : It->second; + + if (Counters.size() != NumBlocks) return false; // wrong number of blocks. + + Counters[0]++; + for (auto BB : CoveredBlocks) + Counters[BB]++; + } + return true; +} + +// Assign weights to each function. +// General principles: +// * any uncovered function gets weight 0. +// * a function with lots of uncovered blocks gets bigger weight. +// * a function with a less frequently executed code gets bigger weight. +Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { + Vector<double> Res(NumFunctions); + for (auto It : Functions) { + auto FunctionID = It.first; + auto Counters = It.second; + assert(FunctionID < NumFunctions); + auto &Weight = Res[FunctionID]; + // Give higher weight if the function has a DFT. + Weight = FunctionsWithDFT.count(FunctionID) ? 1000. : 1; + // Give higher weight to functions with less frequently seen basic blocks. + Weight /= SmallestNonZeroCounter(Counters); + // Give higher weight to functions with the most uncovered basic blocks. + Weight *= NumberOfUncoveredBlocks(Counters) + 1; + } + return Res; +} + +void DataFlowTrace::ReadCoverage(const std::string &DirPath) { + Vector<SizedFile> Files; + GetSizedFilesFromDir(DirPath, &Files); + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; + if (!CorporaHashes.count(Name)) continue; + std::ifstream IF(SF.File); + Coverage.AppendCoverage(IF); + } +} -void DataFlowTrace::Init(const std::string &DirPath, - const std::string &FocusFunction) { - if (DirPath.empty()) return; - const char *kFunctionsTxt = "functions.txt"; +static void DFTStringAppendToVector(Vector<uint8_t> *DFT, + const std::string &DFTString) { + assert(DFT->size() == DFTString.size()); + for (size_t I = 0, Len = DFT->size(); I < Len; I++) + (*DFT)[I] = DFTString[I] == '1'; +} + +// converts a string of '0' and '1' into a Vector<uint8_t> +static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) { + Vector<uint8_t> DFT(DFTString.size()); + DFTStringAppendToVector(&DFT, DFTString); + return DFT; +} + +static bool ParseError(const char *Err, const std::string &Line) { + Printf("DataFlowTrace: parse error: %s: Line: %s\n", Err, Line.c_str()); + return false; +} + +// TODO(metzman): replace std::string with std::string_view for +// better performance. Need to figure our how to use string_view on Windows. +static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, + std::string *DFTString) { + if (!Line.empty() && Line[0] != 'F') + return false; // Ignore coverage. + size_t SpacePos = Line.find(' '); + if (SpacePos == std::string::npos) + return ParseError("no space in the trace line", Line); + if (Line.empty() || Line[0] != 'F') + return ParseError("the trace line doesn't start with 'F'", Line); + *FunctionNum = std::atol(Line.c_str() + 1); + const char *Beg = Line.c_str() + SpacePos + 1; + const char *End = Line.c_str() + Line.size(); + assert(Beg < End); + size_t Len = End - Beg; + for (size_t I = 0; I < Len; I++) { + if (Beg[I] != '0' && Beg[I] != '1') + return ParseError("the trace should contain only 0 or 1", Line); + } + *DFTString = Beg; + return true; +} + +bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Vector<SizedFile> &CorporaFiles, Random &Rand) { + if (DirPath.empty()) return false; Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); Vector<SizedFile> Files; GetSizedFilesFromDir(DirPath, &Files); std::string L; + size_t FocusFuncIdx = SIZE_MAX; + Vector<std::string> FunctionNames; + + // Collect the hashes of the corpus files. + for (auto &SF : CorporaFiles) + CorporaHashes.insert(Hash(FileToVector(SF.File))); // Read functions.txt std::ifstream IF(DirPlusFile(DirPath, kFunctionsTxt)); - size_t FocusFuncIdx = SIZE_MAX; size_t NumFunctions = 0; while (std::getline(IF, L, '\n')) { + FunctionNames.push_back(L); NumFunctions++; - if (FocusFunction == L) + if (*FocusFunction == L) FocusFuncIdx = NumFunctions - 1; } + if (!NumFunctions) + return false; + + if (*FocusFunction == "auto") { + // AUTOFOCUS works like this: + // * reads the coverage data from the DFT files. + // * assigns weights to functions based on coverage. + // * chooses a random function according to the weights. + ReadCoverage(DirPath); + auto Weights = Coverage.FunctionWeights(NumFunctions); + Vector<double> Intervals(NumFunctions + 1); + std::iota(Intervals.begin(), Intervals.end(), 0); + auto Distribution = std::piecewise_constant_distribution<double>( + Intervals.begin(), Intervals.end(), Weights.begin()); + FocusFuncIdx = static_cast<size_t>(Distribution(Rand)); + *FocusFunction = FunctionNames[FocusFuncIdx]; + assert(FocusFuncIdx < NumFunctions); + Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx, + FunctionNames[FocusFuncIdx].c_str()); + for (size_t i = 0; i < NumFunctions; i++) { + if (!Weights[i]) continue; + Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i, + Weights[i], Coverage.GetNumberOfBlocks(i), + Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0), + FunctionNames[i].c_str()); + } + } + if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1) - return; + return false; + // Read traces. size_t NumTraceFiles = 0; size_t NumTracesWithFocusFunction = 0; for (auto &SF : Files) { auto Name = Basename(SF.File); if (Name == kFunctionsTxt) continue; - auto ParseError = [&](const char *Err) { - Printf("DataFlowTrace: parse error: %s\n File: %s\n Line: %s\n", Err, - Name.c_str(), L.c_str()); - }; + if (!CorporaHashes.count(Name)) continue; // not in the corpus. NumTraceFiles++; // Printf("=== %s\n", Name.c_str()); std::ifstream IF(SF.File); while (std::getline(IF, L, '\n')) { - size_t SpacePos = L.find(' '); - if (SpacePos == std::string::npos) - return ParseError("no space in the trace line"); - if (L.empty() || L[0] != 'F') - return ParseError("the trace line doesn't start with 'F'"); - size_t N = std::atol(L.c_str() + 1); - if (N >= NumFunctions) - return ParseError("N is greater than the number of functions"); - if (N == FocusFuncIdx) { + size_t FunctionNum = 0; + std::string DFTString; + if (ParseDFTLine(L, &FunctionNum, &DFTString) && + FunctionNum == FocusFuncIdx) { NumTracesWithFocusFunction++; - const char *Beg = L.c_str() + SpacePos + 1; - const char *End = L.c_str() + L.size(); - assert(Beg < End); - size_t Len = End - Beg; - Vector<uint8_t> V(Len); - for (size_t I = 0; I < Len; I++) { - if (Beg[I] != '0' && Beg[I] != '1') - ParseError("the trace should contain only 0 or 1"); - V[I] = Beg[I] == '1'; - } - Traces[Name] = V; + + if (FunctionNum >= NumFunctions) + return ParseError("N is greater than the number of functions", L); + Traces[Name] = DFTStringToVector(DFTString); // Print just a few small traces. - if (NumTracesWithFocusFunction <= 3 && Len <= 16) - Printf("%s => |%s|\n", Name.c_str(), L.c_str() + SpacePos + 1); - break; // No need to parse the following lines. + if (NumTracesWithFocusFunction <= 3 && DFTString.size() <= 16) + Printf("%s => |%s|\n", Name.c_str(), std::string(DFTString).c_str()); + break; // No need to parse the following lines. } } } - assert(NumTraceFiles == Files.size() - 1); Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, " "%zd traces with focus function\n", NumTraceFiles, NumFunctions, NumTracesWithFocusFunction); + return NumTraceFiles > 0; } -} // namespace fuzzer +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector<SizedFile> &CorporaFiles) { + Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", + DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); + static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0"; + putenv(DFSanEnv); + MkDir(DirPath); + for (auto &F : CorporaFiles) { + // For every input F we need to collect the data flow and the coverage. + // Data flow collection may fail if we request too many DFSan tags at once. + // So, we start from requesting all tags in range [0,Size) and if that fails + // we then request tags in [0,Size/2) and [Size/2, Size), and so on. + // Function number => DFT. + auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File))); + std::unordered_map<size_t, Vector<uint8_t>> DFTMap; + std::unordered_set<std::string> Cov; + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.addArgument(F.File); + Cmd.addArgument(OutPath); + Printf("CMD: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + // Write functions.txt if it's currently empty or doesn't exist. + auto FunctionsTxtPath = DirPlusFile(DirPath, kFunctionsTxt); + if (FileToString(FunctionsTxtPath).empty()) { + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.setOutputFile(FunctionsTxtPath); + ExecuteCommand(Cmd); + } + return 0; +} +} // namespace fuzzer diff --git a/lib/fuzzer/FuzzerDataFlowTrace.h b/lib/fuzzer/FuzzerDataFlowTrace.h index ad4faeab7b2f..d6e3de30a4ef 100644 --- a/lib/fuzzer/FuzzerDataFlowTrace.h +++ b/lib/fuzzer/FuzzerDataFlowTrace.h @@ -1,9 +1,8 @@ //===- FuzzerDataFlowTrace.h - Internal header for the Fuzzer ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::DataFlowTrace; reads and handles a data-flow trace. @@ -30,15 +29,93 @@ #define LLVM_FUZZER_DATA_FLOW_TRACE #include "FuzzerDefs.h" +#include "FuzzerIO.h" #include <unordered_map> +#include <unordered_set> #include <vector> #include <string> namespace fuzzer { + +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector<SizedFile> &CorporaFiles); + +class BlockCoverage { + public: + bool AppendCoverage(std::istream &IN); + bool AppendCoverage(const std::string &S); + + size_t NumCoveredFunctions() const { return Functions.size(); } + + uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + if (BasicBlockId < Counters.size()) + return Counters[BasicBlockId]; + return 0; + } + + uint32_t GetNumberOfBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + return Counters.size(); + } + + uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + uint32_t Result = 0; + for (auto Cnt: Counters) + if (Cnt) + Result++; + return Result; + } + + Vector<double> FunctionWeights(size_t NumFunctions) const; + void clear() { Functions.clear(); } + + private: + + typedef Vector<uint32_t> CoverageVector; + + uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { + uint32_t Res = 0; + for (auto Cnt : Counters) + if (Cnt) + Res++; + return Res; + } + + uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const { + return Counters.size() - NumberOfCoveredBlocks(Counters); + } + + uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const { + assert(!Counters.empty()); + uint32_t Res = Counters[0]; + for (auto Cnt : Counters) + if (Cnt) + Res = Min(Res, Cnt); + assert(Res); + return Res; + } + + // Function ID => vector of counters. + // Each counter represents how many input files trigger the given basic block. + std::unordered_map<size_t, CoverageVector> Functions; + // Functions that have DFT entry. + std::unordered_set<size_t> FunctionsWithDFT; +}; + class DataFlowTrace { public: - void Init(const std::string &DirPath, const std::string &FocusFunction); + void ReadCoverage(const std::string &DirPath); + bool Init(const std::string &DirPath, std::string *FocusFunction, + Vector<SizedFile> &CorporaFiles, Random &Rand); void Clear() { Traces.clear(); } const Vector<uint8_t> *Get(const std::string &InputSha1) const { auto It = Traces.find(InputSha1); @@ -50,6 +127,8 @@ class DataFlowTrace { private: // Input's sha1 => DFT for the FocusFunction. std::unordered_map<std::string, Vector<uint8_t> > Traces; + BlockCoverage Coverage; + std::unordered_set<std::string> CorporaHashes; }; } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerDefs.h b/lib/fuzzer/FuzzerDefs.h index c3dccbcd86f5..320b37d5f8e3 100644 --- a/lib/fuzzer/FuzzerDefs.h +++ b/lib/fuzzer/FuzzerDefs.h @@ -1,9 +1,8 @@ //===- FuzzerDefs.h - Internal header for the Fuzzer ------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Basic definitions. @@ -120,31 +119,39 @@ # define ALWAYS_INLINE #endif // __clang__ -#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) - -#if defined(__has_feature) -# if __has_feature(address_sanitizer) -# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS -# elif __has_feature(memory_sanitizer) -# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY -# else -# define ATTRIBUTE_NO_SANITIZE_ALL -# endif +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_NO_SANITIZE_ADDRESS #else -# define ATTRIBUTE_NO_SANITIZE_ALL +#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) #endif #if LIBFUZZER_WINDOWS +#define ATTRIBUTE_ALIGNED(X) __declspec(align(X)) #define ATTRIBUTE_INTERFACE __declspec(dllexport) // This is used for __sancov_lowest_stack which is needed for // -fsanitize-coverage=stack-depth. That feature is not yet available on // Windows, so make the symbol static to avoid linking errors. -#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ - __attribute__((tls_model("initial-exec"))) thread_local static +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC static +#define ATTRIBUTE_NOINLINE __declspec(noinline) #else +#define ATTRIBUTE_ALIGNED(X) __attribute__((aligned(X))) #define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) #define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local + +#define ATTRIBUTE_NOINLINE __attribute__((noinline)) +#endif + +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS +# elif __has_feature(memory_sanitizer) +# define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY +# else +# define ATTRIBUTE_NO_SANITIZE_ALL +# endif +#else +# define ATTRIBUTE_NO_SANITIZE_ALL #endif namespace fuzzer { diff --git a/lib/fuzzer/FuzzerDictionary.h b/lib/fuzzer/FuzzerDictionary.h index 0d9d91bcd2f1..301c5d9afecf 100644 --- a/lib/fuzzer/FuzzerDictionary.h +++ b/lib/fuzzer/FuzzerDictionary.h @@ -1,9 +1,8 @@ //===- FuzzerDictionary.h - Internal header for the Fuzzer ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::Dictionary diff --git a/lib/fuzzer/FuzzerDriver.cpp b/lib/fuzzer/FuzzerDriver.cpp index ff2a639ac4a3..54c7ff079585 100644 --- a/lib/fuzzer/FuzzerDriver.cpp +++ b/lib/fuzzer/FuzzerDriver.cpp @@ -1,9 +1,8 @@ //===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // FuzzerDriver and flag parsing. @@ -11,12 +10,13 @@ #include "FuzzerCommand.h" #include "FuzzerCorpus.h" +#include "FuzzerFork.h" #include "FuzzerIO.h" #include "FuzzerInterface.h" #include "FuzzerInternal.h" +#include "FuzzerMerge.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" -#include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include <algorithm> #include <atomic> @@ -26,10 +26,16 @@ #include <mutex> #include <string> #include <thread> +#include <fstream> // This function should be present in the libFuzzer so that the client // binary can test for its existence. +#if LIBFUZZER_MSVC +extern "C" void __libfuzzer_is_present() {} +#pragma comment(linker, "/include:__libfuzzer_is_present") +#else extern "C" __attribute__((used)) void __libfuzzer_is_present() {} +#endif // LIBFUZZER_MSVC namespace fuzzer { @@ -176,7 +182,8 @@ static bool ParseOneFlag(const char *Param) { } // We don't use any library to minimize dependencies. -static void ParseFlags(const Vector<std::string> &Args) { +static void ParseFlags(const Vector<std::string> &Args, + const ExternalFunctions *EF) { for (size_t F = 0; F < kNumFlags; F++) { if (FlagDescriptions[F].IntFlag) *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; @@ -186,6 +193,11 @@ static void ParseFlags(const Vector<std::string> &Args) { if (FlagDescriptions[F].StrFlag) *FlagDescriptions[F].StrFlag = nullptr; } + + // Disable len_control by default, if LLVMFuzzerCustomMutator is used. + if (EF->LLVMFuzzerCustomMutator) + Flags.len_control = 0; + Inputs = new Vector<std::string>; for (size_t A = 1; A < Args.size(); A++) { if (ParseOneFlag(Args[A].c_str())) { @@ -316,10 +328,8 @@ int CleanseCrashInput(const Vector<std::string> &Args, assert(Cmd.hasArgument(InputFilePath)); Cmd.removeArgument(InputFilePath); - auto LogFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - auto TmpFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".repro"); + auto LogFilePath = TempPath(".txt"); + auto TmpFilePath = TempPath(".repro"); Cmd.addArgument(TmpFilePath); Cmd.setOutputFile(LogFilePath); Cmd.combineOutAndErr(); @@ -379,8 +389,7 @@ int MinimizeCrashInput(const Vector<std::string> &Args, BaseCmd.addFlag("max_total_time", "600"); } - auto LogFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); + auto LogFilePath = TempPath(".txt"); BaseCmd.setOutputFile(LogFilePath); BaseCmd.combineOutAndErr(); @@ -464,6 +473,34 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { return 0; } +void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, + const Vector<std::string> &Corpora, const char *CFPathOrNull) { + if (Corpora.size() < 2) { + Printf("INFO: Merge requires two or more corpus dirs\n"); + exit(0); + } + + Vector<SizedFile> OldCorpus, NewCorpus; + GetSizedFilesFromDir(Corpora[0], &OldCorpus); + for (size_t i = 1; i < Corpora.size(); i++) + GetSizedFilesFromDir(Corpora[i], &NewCorpus); + std::sort(OldCorpus.begin(), OldCorpus.end()); + std::sort(NewCorpus.begin(), NewCorpus.end()); + + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath(".txt"); + Vector<std::string> NewFiles; + Set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, + {}, &NewCov, CFPath, true); + for (auto &Path : NewFiles) + F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); + // We are done, delete the control file if it was a temporary one. + if (!Flags.merge_control_file) + RemoveFile(CFPath); + + exit(0); +} + int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, UnitVector& Corpus) { Printf("Started dictionary minimization (up to %d tests)\n", @@ -530,6 +567,45 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, return 0; } +Vector<std::string> ParseSeedInuts(const char *seed_inputs) { + // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file + Vector<std::string> Files; + if (!seed_inputs) return Files; + std::string SeedInputs; + if (Flags.seed_inputs[0] == '@') + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. + else + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + if (SeedInputs.empty()) { + Printf("seed_inputs is empty or @file does not exist.\n"); + exit(1); + } + // Parse SeedInputs. + size_t comma_pos = 0; + while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) { + Files.push_back(SeedInputs.substr(comma_pos + 1)); + SeedInputs = SeedInputs.substr(0, comma_pos); + } + Files.push_back(SeedInputs); + return Files; +} + +static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, + const Vector<std::string> &ExtraSeedFiles) { + Vector<SizedFile> SizedFiles; + size_t LastNumFiles = 0; + for (auto &Dir : CorpusDirs) { + GetSizedFilesFromDir(Dir, &SizedFiles); + Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, + Dir.c_str()); + LastNumFiles = SizedFiles.size(); + } + for (auto &File : ExtraSeedFiles) + if (auto Size = FileSize(File)) + SizedFiles.push_back({File, Size}); + return SizedFiles; +} + int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { using namespace fuzzer; assert(argc && argv && "Argument pointers cannot be nullptr"); @@ -546,7 +622,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); exit(1); } - ParseFlags(Args); + ParseFlags(Args, EF); if (Flags.help) { PrintHelp(); return 0; @@ -573,6 +649,9 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.UnitTimeoutSec = Flags.timeout; Options.ErrorExitCode = Flags.error_exitcode; Options.TimeoutExitCode = Flags.timeout_exitcode; + Options.IgnoreTimeouts = Flags.ignore_timeouts; + Options.IgnoreOOMs = Flags.ignore_ooms; + Options.IgnoreCrashes = Flags.ignore_crashes; Options.MaxTotalTimeSec = Flags.max_total_time; Options.DoCrossOver = Flags.cross_over; Options.MutateDepth = Flags.mutate_depth; @@ -609,15 +688,14 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { return 1; if (Flags.verbosity > 0 && !Dictionary.empty()) Printf("Dictionary: %zd entries\n", Dictionary.size()); - bool DoPlainRun = AllInputsAreFiles(); + bool RunIndividualFiles = AllInputsAreFiles(); Options.SaveArtifacts = - !DoPlainRun || Flags.minimize_crash_internal_step; + !RunIndividualFiles || Flags.minimize_crash_internal_step; Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; - Options.DumpCoverage = Flags.dump_coverage; if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; if (Flags.exit_on_item) @@ -626,6 +704,13 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.FocusFunction = Flags.focus_function; if (Flags.data_flow_trace) Options.DataFlowTrace = Flags.data_flow_trace; + if (Flags.features_dir) + Options.FeaturesDir = Flags.features_dir; + if (Flags.collect_data_flow) + Options.CollectDataFlow = Flags.collect_data_flow; + Options.LazyCounters = Flags.lazy_counters; + if (Flags.stop_file) + Options.StopFile = Flags.stop_file; unsigned Seed = Flags.seed; // Initialize Seed. @@ -635,6 +720,15 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); + if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { + if (RunIndividualFiles) + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora({}, *Inputs)); + else + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora(*Inputs, {})); + } + Random Rand(Seed); auto *MD = new MutationDispatcher(Rand, Options); auto *Corpus = new InputCorpus(Options.OutputCorpus); @@ -669,35 +763,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Flags.cleanse_crash) return CleanseCrashInput(Args, Options); -#if 0 // deprecated, to be removed. - if (auto Name = Flags.run_equivalence_server) { - SMR.Destroy(Name); - if (!SMR.Create(Name)) { - Printf("ERROR: can't create shared memory region\n"); - return 1; - } - Printf("INFO: EQUIVALENCE SERVER UP\n"); - while (true) { - SMR.WaitClient(); - size_t Size = SMR.ReadByteArraySize(); - SMR.WriteByteArray(nullptr, 0); - const Unit tmp(SMR.GetByteArray(), SMR.GetByteArray() + Size); - F->ExecuteCallback(tmp.data(), tmp.size()); - SMR.PostServer(); - } - return 0; - } - - if (auto Name = Flags.use_equivalence_server) { - if (!SMR.Open(Name)) { - Printf("ERROR: can't open shared memory region\n"); - return 1; - } - Printf("INFO: EQUIVALENCE CLIENT UP\n"); - } -#endif - - if (DoPlainRun) { + if (RunIndividualFiles) { Options.SaveArtifacts = false; int Runs = std::max(1, Flags.runs); Printf("%s: Running %zd inputs %d time(s) each.\n", ProgName->c_str(), @@ -719,13 +785,11 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } - if (Flags.merge) { - F->CrashResistantMerge(Args, *Inputs, - Flags.load_coverage_summary, - Flags.save_coverage_summary, - Flags.merge_control_file); - exit(0); - } + if (Flags.fork) + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + + if (Flags.merge) + Merge(F, Options, Args, *Inputs, Flags.merge_control_file); if (Flags.merge_inner) { const size_t kDefaultMaxMergeLen = 1 << 20; @@ -757,7 +821,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } - F->Loop(*Inputs); + auto CorporaFiles = ReadCorpora(*Inputs, ParseSeedInuts(Flags.seed_inputs)); + F->Loop(CorporaFiles); if (Flags.verbosity) Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), diff --git a/lib/fuzzer/FuzzerExtFunctions.def b/lib/fuzzer/FuzzerExtFunctions.def index 8bfffdde56d4..41fa0fd2b748 100644 --- a/lib/fuzzer/FuzzerExtFunctions.def +++ b/lib/fuzzer/FuzzerExtFunctions.def @@ -1,9 +1,8 @@ //===- FuzzerExtFunctions.def - External functions --------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This defines the external function pointers that @@ -29,7 +28,7 @@ EXT_FUNC(LLVMFuzzerCustomCrossOver, size_t, EXT_FUNC(__lsan_enable, void, (), false); EXT_FUNC(__lsan_disable, void, (), false); EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false); -EXT_FUNC(__sanitizer_acquire_crash_state, bool, (), true); +EXT_FUNC(__sanitizer_acquire_crash_state, int, (), true); EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int, (void (*malloc_hook)(const volatile void *, size_t), void (*free_hook)(const volatile void *)), @@ -44,8 +43,7 @@ EXT_FUNC(__sanitizer_get_module_and_offset_for_pc, int, size_t module_path_len,void **pc_offset), false); EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true); EXT_FUNC(__sanitizer_set_report_fd, void, (void*), false); -EXT_FUNC(__sanitizer_dump_coverage, void, (const uintptr_t *, uintptr_t), - false); EXT_FUNC(__msan_scoped_disable_interceptor_checks, void, (), false); EXT_FUNC(__msan_scoped_enable_interceptor_checks, void, (), false); EXT_FUNC(__msan_unpoison, void, (const volatile void *, size_t size), false); +EXT_FUNC(__msan_unpoison_param, void, (size_t n), false); diff --git a/lib/fuzzer/FuzzerExtFunctions.h b/lib/fuzzer/FuzzerExtFunctions.h index 2672a385478d..c88aac4e67c2 100644 --- a/lib/fuzzer/FuzzerExtFunctions.h +++ b/lib/fuzzer/FuzzerExtFunctions.h @@ -1,9 +1,8 @@ //===- FuzzerExtFunctions.h - Interface to external functions ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Defines an interface to (possibly optional) functions. diff --git a/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp b/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp index 06bddd5de38f..dcd713459487 100644 --- a/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp +++ b/lib/fuzzer/FuzzerExtFunctionsDlsym.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtFunctionsDlsym.cpp - Interface to external functions ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation for operating systems that support dlsym(). We only use it on diff --git a/lib/fuzzer/FuzzerExtFunctionsWeak.cpp b/lib/fuzzer/FuzzerExtFunctionsWeak.cpp index 6a6ef4932f41..ea5b87bd5196 100644 --- a/lib/fuzzer/FuzzerExtFunctionsWeak.cpp +++ b/lib/fuzzer/FuzzerExtFunctionsWeak.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtFunctionsWeak.cpp - Interface to external functions -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation for Linux. This relies on the linker's support for weak diff --git a/lib/fuzzer/FuzzerExtFunctionsWindows.cpp b/lib/fuzzer/FuzzerExtFunctionsWindows.cpp index b01871439ca0..55efe8f80e90 100644 --- a/lib/fuzzer/FuzzerExtFunctionsWindows.cpp +++ b/lib/fuzzer/FuzzerExtFunctionsWindows.cpp @@ -1,9 +1,8 @@ //=== FuzzerExtWindows.cpp - Interface to external functions --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation of FuzzerExtFunctions for Windows. Uses alternatename when @@ -50,7 +49,7 @@ extern "C" { Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ exit(1); \ } \ - EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG; + EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG #include "FuzzerExtFunctions.def" diff --git a/lib/fuzzer/FuzzerExtraCounters.cpp b/lib/fuzzer/FuzzerExtraCounters.cpp index c99cd89be293..3f38f4fb70c1 100644 --- a/lib/fuzzer/FuzzerExtraCounters.cpp +++ b/lib/fuzzer/FuzzerExtraCounters.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Extra coverage counters defined by user code. diff --git a/lib/fuzzer/FuzzerFlags.def b/lib/fuzzer/FuzzerFlags.def index 91281c979c50..a11cfe4405f3 100644 --- a/lib/fuzzer/FuzzerFlags.def +++ b/lib/fuzzer/FuzzerFlags.def @@ -1,9 +1,8 @@ //===- FuzzerFlags.def - Run-time flags -------------------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Flags. FUZZER_FLAG_INT/FUZZER_FLAG_STRING macros should be defined at the @@ -20,7 +19,10 @@ FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " "then try larger inputs over time. Specifies the rate at which the length " "limit is increased (smaller == faster). If 0, immediately try inputs with " - "size up to max_len.") + "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is used.") +FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " + "to use as an additional seed corpus. Alternatively, an \"@\" followed by " + "the name of a file containing the comma-seperated list.") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") @@ -35,27 +37,26 @@ FUZZER_FLAG_INT( "If one unit runs more than this number of seconds the process will abort.") FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " "this exit code will be used.") -FUZZER_FLAG_INT(timeout_exitcode, 77, "When libFuzzer reports a timeout " +FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout " "this exit code will be used.") FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " + "in a subprocess") +FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") +FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") +FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " "merged into the 1-st corpus. Only interesting units will be taken. " "This flag can be used to minimize a corpus.") +FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") FUZZER_FLAG_STRING(merge_inner, "internal flag") FUZZER_FLAG_STRING(merge_control_file, "Specify a control file used for the merge process. " "If a merge process gets killed it tries to leave this file " "in a state suitable for resuming the merge. " "By default a temporary file will be used.") -FUZZER_FLAG_STRING(save_coverage_summary, "Experimental:" - " save coverage summary to a given file." - " Used with -merge=1") -FUZZER_FLAG_STRING(load_coverage_summary, "Experimental:" - " load coverage summary from a given file." - " Treat this coverage as belonging to the first corpus. " - " Used with -merge=1") FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" " crash input. Use with -runs=N or -max_total_time=N to limit " "the number attempts." @@ -68,6 +69,10 @@ FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" " Use with -exact_artifact_path to specify the output." ) FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag") +FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk." + "Every time a new input is added to the corpus, a corresponding file in the features_dir" + " is created containing the unique features of that input." + " Features are stored in binary format.") FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") FUZZER_FLAG_INT(use_memmem, 1, "Use hints from intercepting memmem, strstr, etc") @@ -107,9 +112,7 @@ FUZZER_FLAG_INT(print_corpus_stats, 0, "If 1, print statistics on corpus elements at exit.") FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" " at exit.") -FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated." - " If 1, dump coverage information as a" - " .sancov file at exit.") +FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") @@ -120,6 +123,9 @@ FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.") FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") +FUZZER_FLAG_INT(lazy_counters, 0, "If 1, a performance optimization is" + "enabled for the 8bit inline counters. " + "Requires that libFuzzer successfully installs its SEGV handler") FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " "if 2, close stderr; if 3, close both. " "Be careful, this will also close e.g. stderr of asan.") @@ -146,10 +152,12 @@ FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " "after this one. Useful for fuzzers that need to do their own " "argument parsing.") FUZZER_FLAG_STRING(focus_function, "Experimental. " - "Fuzzing will focus on inputs that trigger calls to this function") + "Fuzzing will focus on inputs that trigger calls to this function. " + "If -focus_function=auto and -data_flow_trace is used, libFuzzer " + "will choose the focus functions automatically.") -FUZZER_DEPRECATED_FLAG(run_equivalence_server) -FUZZER_DEPRECATED_FLAG(use_equivalence_server) FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_STRING(collect_data_flow, + "Experimental: collect the data flow trace") diff --git a/lib/fuzzer/FuzzerFork.cpp b/lib/fuzzer/FuzzerFork.cpp new file mode 100644 index 000000000000..95ed36551463 --- /dev/null +++ b/lib/fuzzer/FuzzerFork.cpp @@ -0,0 +1,409 @@ +//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Spawn and orchestrate separate fuzzing processes. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerFork.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <fstream> +#include <memory> +#include <mutex> +#include <queue> +#include <sstream> +#include <thread> + +namespace fuzzer { + +struct Stats { + size_t number_of_executed_units = 0; + size_t peak_rss_mb = 0; + size_t average_exec_per_sec = 0; +}; + +static Stats ParseFinalStatsFromLog(const std::string &LogPath) { + std::ifstream In(LogPath); + std::string Line; + Stats Res; + struct { + const char *Name; + size_t *Var; + } NameVarPairs[] = { + {"stat::number_of_executed_units:", &Res.number_of_executed_units}, + {"stat::peak_rss_mb:", &Res.peak_rss_mb}, + {"stat::average_exec_per_sec:", &Res.average_exec_per_sec}, + {nullptr, nullptr}, + }; + while (std::getline(In, Line, '\n')) { + if (Line.find("stat::") != 0) continue; + std::istringstream ISS(Line); + std::string Name; + size_t Val; + ISS >> Name >> Val; + for (size_t i = 0; NameVarPairs[i].Name; i++) + if (Name == NameVarPairs[i].Name) + *NameVarPairs[i].Var = Val; + } + return Res; +} + +struct FuzzJob { + // Inputs. + Command Cmd; + std::string CorpusDir; + std::string FeaturesDir; + std::string LogPath; + std::string SeedListPath; + std::string CFPath; + size_t JobId; + + int DftTimeInSeconds = 0; + + // Fuzzing Outputs. + int ExitCode; + + ~FuzzJob() { + RemoveFile(CFPath); + RemoveFile(LogPath); + RemoveFile(SeedListPath); + RmDirRecursive(CorpusDir); + RmDirRecursive(FeaturesDir); + } +}; + +struct GlobalEnv { + Vector<std::string> Args; + Vector<std::string> CorpusDirs; + std::string MainCorpusDir; + std::string TempDir; + std::string DFTDir; + std::string DataFlowBinary; + Set<uint32_t> Features, Cov; + Set<std::string> FilesWithDFT; + Vector<std::string> Files; + Random *Rand; + std::chrono::system_clock::time_point ProcessStartTime; + int Verbosity = 0; + + size_t NumTimeouts = 0; + size_t NumOOMs = 0; + size_t NumCrashes = 0; + + + size_t NumRuns = 0; + + std::string StopFile() { return DirPlusFile(TempDir, "STOP"); } + + size_t secondsSinceProcessStartUp() const { + return std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::system_clock::now() - ProcessStartTime) + .count(); + } + + FuzzJob *CreateNewJob(size_t JobId) { + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.removeFlag("collect_data_flow"); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. + Cmd.addFlag("print_final_stats", "1"); + Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. + Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId))); + Cmd.addFlag("stop_file", StopFile()); + if (!DataFlowBinary.empty()) { + Cmd.addFlag("data_flow_trace", DFTDir); + if (!Cmd.hasFlag("focus_function")) + Cmd.addFlag("focus_function", "auto"); + } + auto Job = new FuzzJob; + std::string Seeds; + if (size_t CorpusSubsetSize = + std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + auto Time1 = std::chrono::system_clock::now(); + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + auto Time2 = std::chrono::system_clock::now(); + Job->DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count(); + } + if (!Seeds.empty()) { + Job->SeedListPath = + DirPlusFile(TempDir, std::to_string(JobId) + ".seeds"); + WriteToFile(Seeds, Job->SeedListPath); + Cmd.addFlag("seed_inputs", "@" + Job->SeedListPath); + } + Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log"); + Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId)); + Job->FeaturesDir = DirPlusFile(TempDir, "F" + std::to_string(JobId)); + Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge"); + Job->JobId = JobId; + + + Cmd.addArgument(Job->CorpusDir); + Cmd.addFlag("features_dir", Job->FeaturesDir); + + for (auto &D : {Job->CorpusDir, Job->FeaturesDir}) { + RmDirRecursive(D); + MkDir(D); + } + + Cmd.setOutputFile(Job->LogPath); + Cmd.combineOutAndErr(); + + Job->Cmd = Cmd; + + if (Verbosity >= 2) + Printf("Job %zd/%p Created: %s\n", JobId, Job, + Job->Cmd.toString().c_str()); + // Start from very short runs and gradually increase them. + return Job; + } + + void RunOneMergeJob(FuzzJob *Job) { + auto Stats = ParseFinalStatsFromLog(Job->LogPath); + NumRuns += Stats.number_of_executed_units; + + Vector<SizedFile> TempFiles, MergeCandidates; + // Read all newly created inputs and their feature sets. + // Choose only those inputs that have new features. + GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); + std::sort(TempFiles.begin(), TempFiles.end()); + for (auto &F : TempFiles) { + auto FeatureFile = F.File; + FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); + auto FeatureBytes = FileToVector(FeatureFile, 0, false); + assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); + Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); + for (auto Ft : NewFeatures) { + if (!Features.count(Ft)) { + MergeCandidates.push_back(F); + break; + } + } + } + // if (!FilesToAdd.empty() || Job->ExitCode != 0) + Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", + NumRuns, Cov.size(), Features.size(), Files.size(), + Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, + secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); + + if (MergeCandidates.empty()) return; + + Vector<std::string> FilesToAdd; + Set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, + &NewFeatures, Cov, &NewCov, Job->CFPath, false); + for (auto &Path : FilesToAdd) { + auto U = FileToVector(Path); + auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); + WriteToFile(U, NewPath); + Files.push_back(NewPath); + } + Features.insert(NewFeatures.begin(), NewFeatures.end()); + Cov.insert(NewCov.begin(), NewCov.end()); + for (auto Idx : NewCov) + if (auto *TE = TPC.PCTableEntryByIdx(Idx)) + if (TPC.PcIsFuncEntry(TE)) + PrintPC(" NEW_FUNC: %p %F %L\n", "", + TPC.GetNextInstructionPc(TE->PC)); + + } + + + void CollectDFT(const std::string &InputPath) { + if (DataFlowBinary.empty()) return; + if (!FilesWithDFT.insert(InputPath).second) return; + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.addFlag("data_flow_trace", DFTDir); + Cmd.addArgument(InputPath); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.setOutputFile(DirPlusFile(TempDir, "dft.log")); + Cmd.combineOutAndErr(); + // Printf("CollectDFT: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + +}; + +struct JobQueue { + std::queue<FuzzJob *> Qu; + std::mutex Mu; + std::condition_variable Cv; + + void Push(FuzzJob *Job) { + { + std::lock_guard<std::mutex> Lock(Mu); + Qu.push(Job); + } + Cv.notify_one(); + } + FuzzJob *Pop() { + std::unique_lock<std::mutex> Lk(Mu); + // std::lock_guard<std::mutex> Lock(Mu); + Cv.wait(Lk, [&]{return !Qu.empty();}); + assert(!Qu.empty()); + auto Job = Qu.front(); + Qu.pop(); + return Job; + } +}; + +void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { + while (auto Job = FuzzQ->Pop()) { + // Printf("WorkerThread: job %p\n", Job); + Job->ExitCode = ExecuteCommand(Job->Cmd); + MergeQ->Push(Job); + } +} + +// This is just a skeleton of an experimental -fork=1 feature. +void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs) { + Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); + + GlobalEnv Env; + Env.Args = Args; + Env.CorpusDirs = CorpusDirs; + Env.Rand = &Rand; + Env.Verbosity = Options.Verbosity; + Env.ProcessStartTime = std::chrono::system_clock::now(); + Env.DataFlowBinary = Options.CollectDataFlow; + + Vector<SizedFile> SeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &SeedFiles); + std::sort(SeedFiles.begin(), SeedFiles.end()); + Env.TempDir = TempPath(".dir"); + Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); + RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. + MkDir(Env.TempDir); + MkDir(Env.DFTDir); + + + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, + CFPath, false); + RemoveFile(CFPath); + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, + Env.Files.size(), Env.TempDir.c_str()); + + int ExitCode = 0; + + JobQueue FuzzQ, MergeQ; + + auto StopJobs = [&]() { + for (int i = 0; i < NumJobs; i++) + FuzzQ.Push(nullptr); + MergeQ.Push(nullptr); + WriteToFile(Unit({1}), Env.StopFile()); + }; + + size_t JobId = 1; + Vector<std::thread> Threads; + for (int t = 0; t < NumJobs; t++) { + Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + while (true) { + std::unique_ptr<FuzzJob> Job(MergeQ.Pop()); + if (!Job) + break; + ExitCode = Job->ExitCode; + if (ExitCode == Options.InterruptExitCode) { + Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid()); + StopJobs(); + break; + } + Fuzzer::MaybeExitGracefully(); + + Env.RunOneMergeJob(Job.get()); + + // Continue if our crash is one of the ignorred ones. + if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) + Env.NumTimeouts++; + else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) + Env.NumOOMs++; + else if (ExitCode != 0) { + Env.NumCrashes++; + if (Options.IgnoreCrashes) { + std::ifstream In(Job->LogPath); + std::string Line; + while (std::getline(In, Line, '\n')) + if (Line.find("ERROR:") != Line.npos || + Line.find("runtime error:") != Line.npos) + Printf("%s\n", Line.c_str()); + } else { + // And exit if we don't ignore this crash. + Printf("INFO: log from the inner process:\n%s", + FileToString(Job->LogPath).c_str()); + StopJobs(); + break; + } + } + + // Stop if we are over the time budget. + // This is not precise, since other threads are still running + // and we will wait while joining them. + // We also don't stop instantly: other jobs need to finish. + if (Options.MaxTotalTimeSec > 0 && + Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) { + Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n", + Env.secondsSinceProcessStartUp()); + StopJobs(); + break; + } + if (Env.NumRuns >= Options.MaxNumberOfRuns) { + Printf("INFO: fuzzed for %zd iterations, wrapping up soon\n", + Env.NumRuns); + StopJobs(); + break; + } + + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + for (auto &T : Threads) + T.join(); + + // The workers have terminated. Don't try to remove the directory before they + // terminate to avoid a race condition preventing cleanup on Windows. + RmDirRecursive(Env.TempDir); + + // Use the exit code from the last child process. + Printf("INFO: exiting: %d time: %zds\n", ExitCode, + Env.secondsSinceProcessStartUp()); + exit(ExitCode); +} + +} // namespace fuzzer diff --git a/lib/fuzzer/FuzzerFork.h b/lib/fuzzer/FuzzerFork.h new file mode 100644 index 000000000000..b29a43e13fbc --- /dev/null +++ b/lib/fuzzer/FuzzerFork.h @@ -0,0 +1,24 @@ +//===- FuzzerFork.h - run fuzzing in sub-processes --------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_FORK_H +#define LLVM_FUZZER_FORK_H + +#include "FuzzerDefs.h" +#include "FuzzerOptions.h" +#include "FuzzerRandom.h" + +#include <string> + +namespace fuzzer { +void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs); +} // namespace fuzzer + +#endif // LLVM_FUZZER_FORK_H diff --git a/lib/fuzzer/FuzzerIO.cpp b/lib/fuzzer/FuzzerIO.cpp index c4c31e82471f..7e5ba30a2e7d 100644 --- a/lib/fuzzer/FuzzerIO.cpp +++ b/lib/fuzzer/FuzzerIO.cpp @@ -1,17 +1,17 @@ //===- FuzzerIO.cpp - IO utils. -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions. //===----------------------------------------------------------------------===// -#include "FuzzerIO.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerUtil.h" #include <algorithm> #include <cstdarg> #include <fstream> @@ -61,10 +61,19 @@ void CopyFileToErr(const std::string &Path) { } void WriteToFile(const Unit &U, const std::string &Path) { + WriteToFile(U.data(), U.size(), Path); +} + +void WriteToFile(const std::string &Data, const std::string &Path) { + WriteToFile(reinterpret_cast<const uint8_t *>(Data.c_str()), Data.size(), + Path); +} + +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { // Use raw C interface because this function may be called from a sig handler. - FILE *Out = fopen(Path.c_str(), "w"); + FILE *Out = fopen(Path.c_str(), "wb"); if (!Out) return; - fwrite(U.data(), sizeof(U[0]), U.size(), Out); + fwrite(Data, sizeof(Data[0]), Size, Out); fclose(Out); } @@ -126,4 +135,25 @@ void Printf(const char *Fmt, ...) { fflush(OutputFile); } +void VPrintf(bool Verbose, const char *Fmt, ...) { + if (!Verbose) return; + va_list ap; + va_start(ap, Fmt); + vfprintf(OutputFile, Fmt, ap); + va_end(ap); + fflush(OutputFile); +} + +void RmDirRecursive(const std::string &Dir) { + IterateDirRecursive( + Dir, [](const std::string &Path) {}, + [](const std::string &Path) { RmDir(Path); }, + [](const std::string &Path) { RemoveFile(Path); }); +} + +std::string TempPath(const char *Extension) { + return DirPlusFile(TmpDir(), + "libFuzzerTemp." + std::to_string(GetPid()) + Extension); +} + } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerIO.h b/lib/fuzzer/FuzzerIO.h index b4a68190e780..fe0d7b451758 100644 --- a/lib/fuzzer/FuzzerIO.h +++ b/lib/fuzzer/FuzzerIO.h @@ -1,9 +1,8 @@ //===- FuzzerIO.h - Internal header for IO utils ----------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO interface. @@ -25,6 +24,9 @@ std::string FileToString(const std::string &Path); void CopyFileToErr(const std::string &Path); +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path); +// Write Data.c_str() to the file without terminating null character. +void WriteToFile(const std::string &Data, const std::string &Path); void WriteToFile(const Unit &U, const std::string &Path); void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, @@ -40,6 +42,8 @@ std::string DirName(const std::string &FileName); // Returns path to a TmpDir. std::string TmpDir(); +std::string TempPath(const char *Extension); + bool IsInterestingCoverageFile(const std::string &FileName); void DupAndCloseStderr(); @@ -47,6 +51,7 @@ void DupAndCloseStderr(); void CloseStdout(); void Printf(const char *Fmt, ...); +void VPrintf(bool Verbose, const char *Fmt, ...); // Print using raw syscalls, useful when printing at early init stages. void RawPrint(const char *Str); @@ -58,6 +63,16 @@ size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector<std::string> *V, bool TopDir); +void RmDirRecursive(const std::string &Dir); + +// Iterate files and dirs inside Dir, recursively. +// Call DirPreCallback/DirPostCallback on dirs before/after +// calling FileCallback on files. +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)); + struct SizedFile { std::string File; size_t Size; @@ -77,11 +92,17 @@ int CloseFile(int Fd); int DuplicateFile(int Fd); void RemoveFile(const std::string &Path); +void RenameFile(const std::string &OldPath, const std::string &NewPath); void DiscardOutput(int Fd); intptr_t GetHandleFromFd(int fd); +void MkDir(const std::string &Path); +void RmDir(const std::string &Path); + +const std::string &getDevNull(); + } // namespace fuzzer #endif // LLVM_FUZZER_IO_H diff --git a/lib/fuzzer/FuzzerIOPosix.cpp b/lib/fuzzer/FuzzerIOPosix.cpp index 401b4cbbf74f..cfd69bbc8111 100644 --- a/lib/fuzzer/FuzzerIOPosix.cpp +++ b/lib/fuzzer/FuzzerIOPosix.cpp @@ -1,9 +1,8 @@ //===- FuzzerIOPosix.cpp - IO utils for Posix. ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions implementation using Posix API. @@ -79,6 +78,28 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + DirPreCallback(Dir); + DIR *D = opendir(Dir.c_str()); + if (!D) return; + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) + FileCallback(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } + closedir(D); + DirPostCallback(Dir); +} + char GetSeparator() { return '/'; } @@ -99,6 +120,10 @@ void RemoveFile(const std::string &Path) { unlink(Path.c_str()); } +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + void DiscardOutput(int Fd) { FILE* Temp = fopen("/dev/null", "w"); if (!Temp) @@ -137,11 +162,23 @@ bool IsInterestingCoverageFile(const std::string &FileName) { return true; } - void RawPrint(const char *Str) { write(2, Str, strlen(Str)); } +void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); +} + +void RmDir(const std::string &Path) { + rmdir(Path.c_str()); +} + +const std::string &getDevNull() { + static const std::string devNull = "/dev/null"; + return devNull; +} + } // namespace fuzzer #endif // LIBFUZZER_POSIX diff --git a/lib/fuzzer/FuzzerIOWindows.cpp b/lib/fuzzer/FuzzerIOWindows.cpp index 75dcaf72a940..510afebef738 100644 --- a/lib/fuzzer/FuzzerIOWindows.cpp +++ b/lib/fuzzer/FuzzerIOWindows.cpp @@ -1,9 +1,8 @@ //===- FuzzerIOWindows.cpp - IO utils for Windows. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions implementation for Windows. @@ -72,6 +71,11 @@ bool IsFile(const std::string &Path) { return IsFile(Path, Att); } +static bool IsDir(DWORD FileAttrs) { + if (FileAttrs == INVALID_FILE_ATTRIBUTES) return false; + return FileAttrs & FILE_ATTRIBUTE_DIRECTORY; +} + std::string Basename(const std::string &Path) { size_t Pos = Path.find_last_of("/\\"); if (Pos == std::string::npos) return Path; @@ -82,8 +86,10 @@ std::string Basename(const std::string &Path) { size_t FileSize(const std::string &Path) { WIN32_FILE_ATTRIBUTE_DATA attr; if (!GetFileAttributesExA(Path.c_str(), GetFileExInfoStandard, &attr)) { - Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", - Path.c_str(), GetLastError()); + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) + Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), LastError); return 0; } ULARGE_INTEGER size; @@ -141,6 +147,58 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + // TODO(metzman): Implement ListFilesInDirRecursive via this function. + DirPreCallback(Dir); + + DWORD DirAttrs = GetFileAttributesA(Dir.c_str()); + if (!IsDir(DirAttrs)) return; + + std::string TargetDir(Dir); + assert(!TargetDir.empty()); + if (TargetDir.back() != '\\') TargetDir.push_back('\\'); + TargetDir.push_back('*'); + + WIN32_FIND_DATAA FindInfo; + // Find the directory's first file. + HANDLE FindHandle = FindFirstFileA(TargetDir.c_str(), &FindInfo); + if (FindHandle == INVALID_HANDLE_VALUE) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) { + // If the directory isn't empty, then something abnormal is going on. + Printf("FindFirstFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + } + return; + } + + do { + std::string Path = DirPlusFile(Dir, FindInfo.cFileName); + DWORD PathAttrs = FindInfo.dwFileAttributes; + if (IsDir(PathAttrs)) { + // Is Path the current directory (".") or the parent ("..")? + if (strcmp(FindInfo.cFileName, ".") == 0 || + strcmp(FindInfo.cFileName, "..") == 0) + continue; + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } else if (PathAttrs != INVALID_FILE_ATTRIBUTES) { + FileCallback(Path); + } + } while (FindNextFileA(FindHandle, &FindInfo)); + + DWORD LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) + Printf("FindNextFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + + FindClose(FindHandle); + DirPostCallback(Dir); +} + char GetSeparator() { return '\\'; } @@ -161,6 +219,10 @@ void RemoveFile(const std::string &Path) { _unlink(Path.c_str()); } +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + void DiscardOutput(int Fd) { FILE* Temp = fopen("nul", "w"); if (!Temp) @@ -334,8 +396,24 @@ bool IsInterestingCoverageFile(const std::string &FileName) { } void RawPrint(const char *Str) { - // Not tested, may or may not work. Fix if needed. - Printf("%s", Str); + _write(2, Str, strlen(Str)); +} + +void MkDir(const std::string &Path) { + if (CreateDirectoryA(Path.c_str(), nullptr)) return; + Printf("CreateDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +void RmDir(const std::string &Path) { + if (RemoveDirectoryA(Path.c_str())) return; + Printf("RemoveDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +const std::string &getDevNull() { + static const std::string devNull = "NUL"; + return devNull; } } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerInterface.h b/lib/fuzzer/FuzzerInterface.h index 0f7effb2ab6a..4f62822eac05 100644 --- a/lib/fuzzer/FuzzerInterface.h +++ b/lib/fuzzer/FuzzerInterface.h @@ -1,9 +1,8 @@ //===- FuzzerInterface.h - Interface header for the Fuzzer ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Define the interface between libFuzzer and the library being tested. @@ -26,25 +25,32 @@ extern "C" { #endif // __cplusplus +// Define FUZZER_INTERFACE_VISIBILITY to set default visibility in a way that +// doesn't break MSVC. +#if defined(_WIN32) +#define FUZZER_INTERFACE_VISIBILITY __declspec(dllexport) +#else +#define FUZZER_INTERFACE_VISIBILITY __attribute__((visibility("default"))) +#endif + // Mandatory user-provided target function. // Executes the code under test with [Data, Data+Size) as the input. // libFuzzer will invoke this function *many* times with different inputs. // Must return 0. -__attribute__((visibility("default"))) int +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); // Optional user-provided initialization function. // If provided, this function will be called by libFuzzer once at startup. // It may read and modify argc/argv. // Must return 0. -__attribute__((visibility("default"))) int LLVMFuzzerInitialize(int *argc, - char ***argv); +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerInitialize(int *argc, char ***argv); // Optional user-provided custom mutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. // Given the same Seed produces the same mutation. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, unsigned int Seed); @@ -52,7 +58,7 @@ LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, // Combines pieces of Data1 & Data2 together into Out. // Returns the new size, which is not greater than MaxOutSize. // Should produce the same mutation given the same Seed. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2, size_t Size2, uint8_t *Out, size_t MaxOutSize, unsigned int Seed); @@ -61,9 +67,11 @@ LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, // libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize); +#undef FUZZER_INTERFACE_VISIBILITY + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/lib/fuzzer/FuzzerInternal.h b/lib/fuzzer/FuzzerInternal.h index a7fdc89cba5f..f2a4c437de38 100644 --- a/lib/fuzzer/FuzzerInternal.h +++ b/lib/fuzzer/FuzzerInternal.h @@ -1,9 +1,8 @@ //===- FuzzerInternal.h - Internal header for the Fuzzer --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Define the main class fuzzer::Fuzzer and most functions. @@ -36,8 +35,8 @@ public: Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options); ~Fuzzer(); - void Loop(const Vector<std::string> &CorpusDirs); - void ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs); + void Loop(Vector<SizedFile> &CorporaFiles); + void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles); void MinimizeCrashLoop(const Unit &U); void RereadOutputCorpus(size_t MaxSize); @@ -72,11 +71,6 @@ public: // Merge Corpora[1:] into Corpora[0]. void Merge(const Vector<std::string> &Corpora); - void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<std::string> &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull, - const char *MergeControlFilePathOrNull); void CrashResistantMergeInternalStep(const std::string &ControlFilePath); MutationDispatcher &GetMD() { return MD; } void PrintFinalStats(); @@ -90,20 +84,19 @@ public: bool DuringInitialCorpusExecution); void HandleMalloc(size_t Size); - void AnnounceOutput(const uint8_t *Data, size_t Size); + static void MaybeExitGracefully(); + std::string WriteToOutputCorpus(const Unit &U); private: void AlarmCallback(); void CrashCallback(); void ExitCallback(); - void MaybeExitGracefully(); void CrashOnOverwrittenData(); void InterruptCallback(); void MutateAndTestOne(); void PurgeAllocator(); void ReportNewCoverage(InputInfo *II, const Unit &U); void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); - void WriteToOutputCorpus(const Unit &U); void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0); void PrintStatusForNewUnit(const Unit &U, const char *Text); diff --git a/lib/fuzzer/FuzzerLoop.cpp b/lib/fuzzer/FuzzerLoop.cpp index a32a30723004..f773f9a13398 100644 --- a/lib/fuzzer/FuzzerLoop.cpp +++ b/lib/fuzzer/FuzzerLoop.cpp @@ -1,9 +1,8 @@ //===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Fuzzer's main loop. @@ -14,7 +13,6 @@ #include "FuzzerInternal.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" -#include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include <algorithm> #include <cstring> @@ -41,8 +39,6 @@ static const size_t kMaxUnitSizeToPrint = 256; thread_local bool Fuzzer::IsMyThread; -SharedMemoryRegion SMR; - bool RunningUserCallback = false; // Only one Fuzzer per process. @@ -135,7 +131,7 @@ void Fuzzer::HandleMalloc(size_t Size) { DumpCurrentUnit("oom-"); Printf("SUMMARY: libFuzzer: out-of-memory\n"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // Stop right now. + _Exit(Options.OOMExitCode); // Stop right now. } Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, @@ -157,12 +153,10 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec) EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus); MaxInputLen = MaxMutationLen = Options.MaxLen; - TmpMaxMutationLen = Max(size_t(4), Corpus.MaxInputSize()); + TmpMaxMutationLen = 0; // Will be set once we load the corpus. AllocateCurrentUnitData(); CurrentUnitSize = 0; memset(BaseSha1, 0, sizeof(BaseSha1)); - TPC.SetFocusFunction(Options.FocusFunction); - DFT.Init(Options.DataFlowTrace, Options.FocusFunction); } Fuzzer::~Fuzzer() {} @@ -231,8 +225,9 @@ void Fuzzer::StaticFileSizeExceedCallback() { } void Fuzzer::CrashCallback() { - if (EF->__sanitizer_acquire_crash_state) - EF->__sanitizer_acquire_crash_state(); + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid()); PrintStackTrace(); Printf("NOTE: libFuzzer has rudimentary signal handlers.\n" @@ -259,16 +254,20 @@ void Fuzzer::ExitCallback() { } void Fuzzer::MaybeExitGracefully() { - if (!GracefulExitRequested) return; + if (!F->GracefulExitRequested) return; Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); - PrintFinalStats(); + RmDirRecursive(TempPath(".dir")); + F->PrintFinalStats(); _Exit(0); } void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); - _Exit(0); // Stop right now, don't perform any at-exit actions. + ScopedDisableMsanInterceptorChecks S; // RmDirRecursive may call opendir(). + RmDirRecursive(TempPath(".dir")); + // Stop right now, don't perform any at-exit actions. + _Exit(Options.InterruptExitCode); } NO_SANITIZE_MEMORY @@ -317,7 +316,7 @@ void Fuzzer::RssLimitCallback() { DumpCurrentUnit("oom-"); Printf("SUMMARY: libFuzzer: out-of-memory\n"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // Stop right now. + _Exit(Options.OOMExitCode); // Stop right now. } void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { @@ -355,8 +354,6 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { void Fuzzer::PrintFinalStats() { if (Options.PrintCoverage) TPC.PrintCoverage(); - if (Options.DumpCoverage) - TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); if (!Options.PrintFinalStats) @@ -388,10 +385,10 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { static auto *PCsSet = new Set<uintptr_t>; - auto HandlePC = [&](uintptr_t PC) { - if (!PCsSet->insert(PC).second) + auto HandlePC = [&](const TracePC::PCTableEntry *TE) { + if (!PCsSet->insert(TE->PC).second) return; - std::string Descr = DescribePC("%F %L", PC + 1); + std::string Descr = DescribePC("%F %L", TE->PC + 1); if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) { Printf("INFO: found line matching '%s', exiting.\n", Options.ExitOnSrcPos.c_str()); @@ -447,6 +444,23 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { } } +static void WriteFeatureSetToFile(const std::string &FeaturesDir, + const std::string &FileName, + const Vector<uint32_t> &FeatureSet) { + if (FeaturesDir.empty() || FeatureSet.empty()) return; + WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()), + FeatureSet.size() * sizeof(FeatureSet[0]), + DirPlusFile(FeaturesDir, FileName)); +} + +static void RenameFeatureSetFile(const std::string &FeaturesDir, + const std::string &OldFile, + const std::string &NewFile) { + if (FeaturesDir.empty()) return; + RenameFile(DirPlusFile(FeaturesDir, OldFile), + DirPlusFile(FeaturesDir, NewFile)); +} + bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, InputInfo *II, bool *FoundUniqFeatures) { if (!Size) @@ -471,15 +485,21 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; if (NumNewFeatures) { TPC.UpdateObservedPCs(); - Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, - TPC.ObservedFocusFunction(), UniqFeatureSetTmp, DFT, II); + auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, + MayDeleteFile, TPC.ObservedFocusFunction(), + UniqFeatureSetTmp, DFT, II); + WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1), + NewII->UniqFeatureSet); return true; } if (II && FoundUniqFeaturesOfII && II->DataFlowTraceForFocusFunction.empty() && FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && II->U.size() > Size) { + auto OldFeaturesFile = Sha1ToString(II->Sha1); Corpus.Replace(II, {Data, Data + Size}); + RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, + Sha1ToString(II->Sha1)); return true; } return false; @@ -513,14 +533,14 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); - if (SMR.IsClient()) - SMR.WriteByteArray(Data, Size); // We copy the contents of Unit into a separate heap buffer // so that we reliably find buffer overflows in it. uint8_t *DataCopy = new uint8_t[Size]; memcpy(DataCopy, Data, Size); if (EF->__msan_unpoison) EF->__msan_unpoison(DataCopy, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(2); if (CurrentUnitData && CurrentUnitData != Data) memcpy(CurrentUnitData, Data, Size); CurrentUnitSize = Size; @@ -543,15 +563,16 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { delete[] DataCopy; } -void Fuzzer::WriteToOutputCorpus(const Unit &U) { +std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { if (Options.OnlyASCII) assert(IsASCII(U)); if (Options.OutputCorpus.empty()) - return; + return ""; std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); WriteToFile(U, Path); if (Options.Verbosity >= 2) Printf("Written %zd bytes to %s\n", U.size(), Path.c_str()); + return Path; } void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) { @@ -637,6 +658,8 @@ void Fuzzer::MutateAndTestOne() { MD.StartMutationSequence(); auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); + if (Options.DoCrossOver) + MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U); const auto &U = II.U; memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); assert(CurrentUnitData); @@ -659,7 +682,9 @@ void Fuzzer::MutateAndTestOne() { Size <= CurrentMaxMutationLen) NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, II.DataFlowTraceForFocusFunction); - else + + // If MutateWithMask either failed or wasn't called, call default Mutate. + if (!NewSize) NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); assert(NewSize > 0 && "Mutator returned empty unit"); assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); @@ -676,7 +701,7 @@ void Fuzzer::MutateAndTestOne() { break; // We will mutate this input more in the next rounds. } if (Options.ReduceDepth && !FoundUniqFeatures) - break; + break; } } @@ -695,21 +720,13 @@ void Fuzzer::PurgeAllocator() { LastAllocatorPurgeAttemptTime = system_clock::now(); } -void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { +void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { const size_t kMaxSaneLen = 1 << 20; const size_t kMinDefaultLen = 4096; - Vector<SizedFile> SizedFiles; size_t MaxSize = 0; size_t MinSize = -1; size_t TotalSize = 0; - size_t LastNumFiles = 0; - for (auto &Dir : CorpusDirs) { - GetSizedFilesFromDir(Dir, &SizedFiles); - Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, - Dir.c_str()); - LastNumFiles = SizedFiles.size(); - } - for (auto &File : SizedFiles) { + for (auto &File : CorporaFiles) { MaxSize = Max(File.Size, MaxSize); MinSize = Min(File.Size, MinSize); TotalSize += File.Size; @@ -722,24 +739,28 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { uint8_t dummy = 0; ExecuteCallback(&dummy, 0); - if (SizedFiles.empty()) { + // Protect lazy counters here, after the once-init code has been executed. + if (Options.LazyCounters) + TPC.ProtectLazyCounters(); + + if (CorporaFiles.empty()) { Printf("INFO: A corpus is not provided, starting from an empty corpus\n"); Unit U({'\n'}); // Valid ASCII input. RunOne(U.data(), U.size()); } else { Printf("INFO: seed corpus: files: %zd min: %zdb max: %zdb total: %zdb" " rss: %zdMb\n", - SizedFiles.size(), MinSize, MaxSize, TotalSize, GetPeakRSSMb()); + CorporaFiles.size(), MinSize, MaxSize, TotalSize, GetPeakRSSMb()); if (Options.ShuffleAtStartUp) - std::shuffle(SizedFiles.begin(), SizedFiles.end(), MD.GetRand()); + std::shuffle(CorporaFiles.begin(), CorporaFiles.end(), MD.GetRand()); if (Options.PreferSmall) { - std::stable_sort(SizedFiles.begin(), SizedFiles.end()); - assert(SizedFiles.front().Size <= SizedFiles.back().Size); + std::stable_sort(CorporaFiles.begin(), CorporaFiles.end()); + assert(CorporaFiles.front().Size <= CorporaFiles.back().Size); } // Load and execute inputs one by one. - for (auto &SF : SizedFiles) { + for (auto &SF : CorporaFiles) { auto U = FileToVector(SF.File, MaxInputLen, /*ExitOnError=*/false); assert(U.size() <= MaxInputLen); RunOne(U.data(), U.size()); @@ -764,16 +785,25 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { } } -void Fuzzer::Loop(const Vector<std::string> &CorpusDirs) { - ReadAndExecuteSeedCorpora(CorpusDirs); +void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { + auto FocusFunctionOrAuto = Options.FocusFunction; + DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, + MD.GetRand()); + TPC.SetFocusFunction(FocusFunctionOrAuto); + ReadAndExecuteSeedCorpora(CorporaFiles); DFT.Clear(); // No need for DFT any more. TPC.SetPrintNewPCs(Options.PrintNewCovPcs); TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); system_clock::time_point LastCorpusReload = system_clock::now(); - if (Options.DoCrossOver) - MD.SetCorpus(&Corpus); + + TmpMaxMutationLen = + Min(MaxMutationLen, Max(size_t(4), Corpus.MaxInputSize())); + while (true) { auto Now = system_clock::now(); + if (!Options.StopFile.empty() && + !FileToVector(Options.StopFile, 1, false).empty()) + break; if (duration_cast<seconds>(Now - LastCorpusReload).count() >= Options.ReloadIntervalSec) { RereadOutputCorpus(MaxInputLen); @@ -824,44 +854,14 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) { } } -void Fuzzer::AnnounceOutput(const uint8_t *Data, size_t Size) { - if (SMR.IsServer()) { - SMR.WriteByteArray(Data, Size); - } else if (SMR.IsClient()) { - SMR.PostClient(); - SMR.WaitServer(); - size_t OtherSize = SMR.ReadByteArraySize(); - uint8_t *OtherData = SMR.GetByteArray(); - if (Size != OtherSize || memcmp(Data, OtherData, Size) != 0) { - size_t i = 0; - for (i = 0; i < Min(Size, OtherSize); i++) - if (Data[i] != OtherData[i]) - break; - Printf("==%lu== ERROR: libFuzzer: equivalence-mismatch. Sizes: %zd %zd; " - "offset %zd\n", - GetPid(), Size, OtherSize, i); - DumpCurrentUnit("mismatch-"); - Printf("SUMMARY: libFuzzer: equivalence-mismatch\n"); - PrintFinalStats(); - _Exit(Options.ErrorExitCode); - } - } -} - } // namespace fuzzer extern "C" { -__attribute__((visibility("default"))) size_t +ATTRIBUTE_INTERFACE size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { assert(fuzzer::F); return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize); } -// Experimental -__attribute__((visibility("default"))) void -LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) { - assert(fuzzer::F); - fuzzer::F->AnnounceOutput(Data, Size); -} } // extern "C" diff --git a/lib/fuzzer/FuzzerMain.cpp b/lib/fuzzer/FuzzerMain.cpp index f2c8e9c7bb11..771a34aed316 100644 --- a/lib/fuzzer/FuzzerMain.cpp +++ b/lib/fuzzer/FuzzerMain.cpp @@ -1,9 +1,8 @@ //===- FuzzerMain.cpp - main() function and flags -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // main() and flags. @@ -16,6 +15,6 @@ extern "C" { int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); } // extern "C" -__attribute__((visibility("default"))) int main(int argc, char **argv) { +ATTRIBUTE_INTERFACE int main(int argc, char **argv) { return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput); } diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 5f3052a39c16..75b2b5d59b9c 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -1,9 +1,8 @@ //===- FuzzerMerge.cpp - merging corpora ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Merging corpora. @@ -43,10 +42,12 @@ void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { // file1 // file2 # One file name per line. // STARTED 0 123 # FileID, file size -// DONE 0 1 4 6 8 # FileID COV1 COV2 ... -// STARTED 1 456 # If DONE is missing, the input crashed while processing. +// FT 0 1 4 6 8 # FileID COV1 COV2 ... +// COV 0 7 8 9 # FileID COV1 COV1 +// STARTED 1 456 # If FT is missing, the input crashed while processing. // STARTED 2 567 -// DONE 2 8 9 +// FT 2 8 9 +// COV 2 11 12 bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastFailure.clear(); std::string Line; @@ -71,11 +72,12 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { if (!std::getline(IS, Files[i].Name, '\n')) return false; - // Parse STARTED and DONE lines. + // Parse STARTED, FT, and COV lines. size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; Vector<uint32_t> TmpFeatures; + Set<uint32_t> PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; @@ -90,19 +92,25 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = ExpectedStartMarker; assert(ExpectedStartMarker < Files.size()); ExpectedStartMarker++; - } else if (Marker == "DONE") { - // DONE FILE_ID COV1 COV2 COV3 ... + } else if (Marker == "FT") { + // FT FILE_ID COV1 COV2 COV3 ... size_t CurrentFileIdx = N; if (CurrentFileIdx != LastSeenStartMarker) return false; LastSeenStartMarker = kInvalidStartMarker; if (ParseCoverage) { TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. - while (ISS1 >> std::hex >> N) + while (ISS1 >> N) TmpFeatures.push_back(N); std::sort(TmpFeatures.begin(), TmpFeatures.end()); Files[CurrentFileIdx].Features = TmpFeatures; } + } else if (Marker == "COV") { + size_t CurrentFileIdx = N; + if (ParseCoverage) + while (ISS1 >> N) + if (PCs.insert(N).second) + Files[CurrentFileIdx].Cov.push_back(N); } else { return false; } @@ -121,21 +129,21 @@ size_t Merger::ApproximateMemoryConsumption() const { return Res; } -// Decides which files need to be merged (add thost to NewFiles). +// Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, Vector<std::string> *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - Set<uint32_t> AllFeatures(InitialFeatures); + Set<uint32_t> AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { auto &Cur = Files[i].Features; AllFeatures.insert(Cur.begin(), Cur.end()); } - size_t InitialNumFeatures = AllFeatures.size(); - // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; @@ -161,22 +169,20 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, auto &Cur = Files[i].Features; // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(), // Files[i].Size, Cur.size()); - size_t OldSize = AllFeatures.size(); - AllFeatures.insert(Cur.begin(), Cur.end()); - if (AllFeatures.size() > OldSize) + bool FoundNewFeatures = false; + for (auto Fe: Cur) { + if (AllFeatures.insert(Fe).second) { + FoundNewFeatures = true; + NewFeatures->insert(Fe); + } + } + if (FoundNewFeatures) NewFiles->push_back(Files[i].Name); + for (auto Cov : Files[i].Cov) + if (InitialCov.find(Cov) == InitialCov.end()) + NewCov->insert(Cov); } - return AllFeatures.size() - InitialNumFeatures; -} - -void Merger::PrintSummary(std::ostream &OS) { - for (auto &File : Files) { - OS << std::hex; - OS << File.Name << " size: " << File.Size << " features: "; - for (auto Feature : File.Features) - OS << " " << Feature; - OS << "\n"; - } + return NewFeatures->size(); } Set<uint32_t> Merger::AllFeatures() const { @@ -186,25 +192,6 @@ Set<uint32_t> Merger::AllFeatures() const { return S; } -Set<uint32_t> Merger::ParseSummary(std::istream &IS) { - std::string Line, Tmp; - Set<uint32_t> Res; - while (std::getline(IS, Line, '\n')) { - size_t N; - std::istringstream ISS1(Line); - ISS1 >> Tmp; // Name - ISS1 >> Tmp; // size: - assert(Tmp == "size:" && "Corrupt summary file"); - ISS1 >> std::hex; - ISS1 >> N; // File Size - ISS1 >> Tmp; // features: - assert(Tmp == "features:" && "Corrupt summary file"); - while (ISS1 >> std::hex >> N) - Res.insert(N); - } - return Res; -} - // Inner process. May crash if the target crashes. void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); @@ -223,8 +210,9 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); Set<size_t> AllFeatures; + Set<const TracePC::PCTableEntry *> AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { - MaybeExitGracefully(); + Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); if (U.size() > MaxInputLen) { U.resize(MaxInputLen); @@ -232,7 +220,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { } std::ostringstream StartedLine; // Write the pre-run marker. - OF << "STARTED " << std::dec << i << " " << U.size() << "\n"; + OF << "STARTED " << i << " " << U.size() << "\n"; OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); @@ -247,26 +235,36 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { if (AllFeatures.insert(Feature).second) UniqFeatures.insert(Feature); }); + TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) PrintStats("pulse "); // Write the post-run marker and the coverage. - OF << "DONE " << i; + OF << "FT " << i; for (size_t F : UniqFeatures) - OF << " " << std::hex << F; + OF << " " << F; + OF << "\n"; + OF << "COV " << i; + TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { + if (AllPCs.insert(TE).second) + OF << " " << TPC.PCTableEntryIdx(TE); + }); OF << "\n"; OF.flush(); } + PrintStats("DONE "); } static void WriteNewControlFile(const std::string &CFPath, - const Vector<SizedFile> &AllFiles, - size_t NumFilesInFirstCorpus) { + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus) { RemoveFile(CFPath); std::ofstream ControlFile(CFPath); - ControlFile << AllFiles.size() << "\n"; - ControlFile << NumFilesInFirstCorpus << "\n"; - for (auto &SF: AllFiles) + ControlFile << (OldCorpus.size() + NewCorpus.size()) << "\n"; + ControlFile << OldCorpus.size() << "\n"; + for (auto &SF: OldCorpus) + ControlFile << SF.File << "\n"; + for (auto &SF: NewCorpus) ControlFile << SF.File << "\n"; if (!ControlFile) { Printf("MERGE-OUTER: failed to write to the control file: %s\n", @@ -275,116 +273,90 @@ static void WriteNewControlFile(const std::string &CFPath, } } -// Outer process. Does not call the target code and thus sohuld not fail. -void Fuzzer::CrashResistantMerge(const Vector<std::string> &Args, - const Vector<std::string> &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull, - const char *MergeControlFilePathOrNull) { - if (Corpora.size() <= 1) { - Printf("Merge requires two or more corpus dirs\n"); - return; - } - auto CFPath = - MergeControlFilePathOrNull - ? MergeControlFilePathOrNull - : DirPlusFile(TmpDir(), - "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - +// Outer process. Does not call the target code and thus should not fail. +void CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, + const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, + Set<uint32_t> *NewCov, + const std::string &CFPath, + bool V /*Verbose*/) { + if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; - if (MergeControlFilePathOrNull && FileSize(MergeControlFilePathOrNull)) { - Printf("MERGE-OUTER: non-empty control file provided: '%s'\n", - MergeControlFilePathOrNull); + if (FileSize(CFPath)) { + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", + CFPath.c_str()); Merger M; - std::ifstream IF(MergeControlFilePathOrNull); + std::ifstream IF(CFPath); if (M.Parse(IF, /*ParseCoverage=*/false)) { - Printf("MERGE-OUTER: control file ok, %zd files total," + VPrintf(V, "MERGE-OUTER: control file ok, %zd files total," " first not processed file %zd\n", M.Files.size(), M.FirstNotProcessedFile); if (!M.LastFailure.empty()) - Printf("MERGE-OUTER: '%s' will be skipped as unlucky " + VPrintf(V, "MERGE-OUTER: '%s' will be skipped as unlucky " "(merge has stumbled on it the last time)\n", M.LastFailure.c_str()); if (M.FirstNotProcessedFile >= M.Files.size()) { - Printf("MERGE-OUTER: nothing to do, merge has been completed before\n"); + VPrintf( + V, "MERGE-OUTER: nothing to do, merge has been completed before\n"); exit(0); } NumAttempts = M.Files.size() - M.FirstNotProcessedFile; } else { - Printf("MERGE-OUTER: bad control file, will overwrite it\n"); + VPrintf(V, "MERGE-OUTER: bad control file, will overwrite it\n"); } } if (!NumAttempts) { // The supplied control file is empty or bad, create a fresh one. - Vector<SizedFile> AllFiles; - GetSizedFilesFromDir(Corpora[0], &AllFiles); - size_t NumFilesInFirstCorpus = AllFiles.size(); - std::sort(AllFiles.begin(), AllFiles.end()); - for (size_t i = 1; i < Corpora.size(); i++) - GetSizedFilesFromDir(Corpora[i], &AllFiles); - std::sort(AllFiles.begin() + NumFilesInFirstCorpus, AllFiles.end()); - Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", - AllFiles.size(), NumFilesInFirstCorpus); - WriteNewControlFile(CFPath, AllFiles, NumFilesInFirstCorpus); - NumAttempts = AllFiles.size(); + NumAttempts = OldCorpus.size() + NewCorpus.size(); + VPrintf(V, "MERGE-OUTER: %zd files, %zd in the initial corpus\n", + NumAttempts, OldCorpus.size()); + WriteNewControlFile(CFPath, OldCorpus, NewCorpus); } // Execute the inner process until it passes. // Every inner process should execute at least one input. Command BaseCmd(Args); BaseCmd.removeFlag("merge"); - bool Success = false; + BaseCmd.removeFlag("fork"); + BaseCmd.removeFlag("collect_data_flow"); for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { - MaybeExitGracefully(); - Printf("MERGE-OUTER: attempt %zd\n", Attempt); + Fuzzer::MaybeExitGracefully(); + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); Cmd.addFlag("merge_inner", "1"); + if (!V) { + Cmd.setOutputFile(getDevNull()); + Cmd.combineOutAndErr(); + } auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); - Success = true; + VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); break; } } - if (!Success) { - Printf("MERGE-OUTER: zero succesfull attempts, exiting\n"); - exit(1); - } // Read the control file and do the merge. Merger M; std::ifstream IF(CFPath); IF.seekg(0, IF.end); - Printf("MERGE-OUTER: the control file has %zd bytes\n", (size_t)IF.tellg()); + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); IF.seekg(0, IF.beg); M.ParseOrExit(IF, true); IF.close(); - Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", - M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); - if (CoverageSummaryOutputPathOrNull) { - Printf("MERGE-OUTER: writing coverage summary for %zd files to %s\n", - M.Files.size(), CoverageSummaryOutputPathOrNull); - std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull); - M.PrintSummary(SummaryOut); - } - Vector<std::string> NewFiles; - Set<uint32_t> InitialFeatures; - if (CoverageSummaryInputPathOrNull) { - std::ifstream SummaryIn(CoverageSummaryInputPathOrNull); - InitialFeatures = M.ParseSummary(SummaryIn); - Printf("MERGE-OUTER: coverage summary loaded from %s, %zd features found\n", - CoverageSummaryInputPathOrNull, InitialFeatures.size()); - } - size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles); - Printf("MERGE-OUTER: %zd new files with %zd new features added\n", - NewFiles.size(), NumNewFeatures); - for (auto &F: NewFiles) - WriteToOutputCorpus(FileToVector(F, MaxInputLen)); - // We are done, delete the control file if it was a temporary one. - if (!MergeControlFilePathOrNull) - RemoveFile(CFPath); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", + M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); } } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerMerge.h b/lib/fuzzer/FuzzerMerge.h index e54885a1ebae..c14dd589e62d 100644 --- a/lib/fuzzer/FuzzerMerge.h +++ b/lib/fuzzer/FuzzerMerge.h @@ -1,9 +1,8 @@ //===- FuzzerMerge.h - merging corpa ----------------------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Merging Corpora. @@ -52,7 +51,7 @@ namespace fuzzer { struct MergeFileInfo { std::string Name; size_t Size = 0; - Vector<uint32_t> Features; + Vector<uint32_t> Features, Cov; }; struct Merger { @@ -64,17 +63,24 @@ struct Merger { bool Parse(std::istream &IS, bool ParseCoverage); bool Parse(const std::string &Str, bool ParseCoverage); void ParseOrExit(std::istream &IS, bool ParseCoverage); - void PrintSummary(std::ostream &OS); - Set<uint32_t> ParseSummary(std::istream &IS); - size_t Merge(const Set<uint32_t> &InitialFeatures, + size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, Vector<std::string> *NewFiles); - size_t Merge(Vector<std::string> *NewFiles) { - return Merge(Set<uint32_t>{}, NewFiles); - } size_t ApproximateMemoryConsumption() const; Set<uint32_t> AllFeatures() const; }; +void CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, + const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, + Set<uint32_t> *NewCov, + const std::string &CFPath, + bool Verbose); + } // namespace fuzzer #endif // LLVM_FUZZER_MERGE_H diff --git a/lib/fuzzer/FuzzerMutate.cpp b/lib/fuzzer/FuzzerMutate.cpp index 142b2b0b0014..29541eac5dc6 100644 --- a/lib/fuzzer/FuzzerMutate.cpp +++ b/lib/fuzzer/FuzzerMutate.cpp @@ -1,20 +1,19 @@ //===- FuzzerMutate.cpp - Mutate a test input -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Mutate a test input. //===----------------------------------------------------------------------===// -#include "FuzzerMutate.h" -#include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" +#include "FuzzerMutate.h" #include "FuzzerOptions.h" +#include "FuzzerTracePC.h" namespace fuzzer { @@ -73,10 +72,10 @@ size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { - if (!Corpus || Corpus->size() < 2 || Size == 0) + if (Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &Other = (*Corpus)[Idx]; + if (!CrossOverWith) return 0; + const Unit &Other = *CrossOverWith; if (Other.empty()) return 0; CustomCrossOverInPlaceHere.resize(MaxSize); @@ -422,9 +421,9 @@ size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { if (Size > MaxSize) return 0; - if (!Corpus || Corpus->size() < 2 || Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &O = (*Corpus)[Idx]; + if (Size == 0) return 0; + if (!CrossOverWith) return 0; + const Unit &O = *CrossOverWith; if (O.empty()) return 0; MutateInPlaceHere.resize(MaxSize); auto &U = MutateInPlaceHere; @@ -530,7 +529,7 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, const Vector<uint8_t> &Mask) { - assert(Size <= Mask.size()); + size_t MaskedSize = std::min(Size, Mask.size()); // * Copy the worthy bytes into a temporary array T // * Mutate T // * Copy T back. @@ -539,16 +538,17 @@ size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, if (T.size() < Size) T.resize(Size); size_t OneBits = 0; - for (size_t I = 0; I < Size; I++) + for (size_t I = 0; I < MaskedSize; I++) if (Mask[I]) T[OneBits++] = Data[I]; + if (!OneBits) return 0; assert(!T.empty()); size_t NewSize = Mutate(T.data(), OneBits, OneBits); assert(NewSize <= OneBits); (void)NewSize; // Even if NewSize < OneBits we still use all OneBits bytes. - for (size_t I = 0, J = 0; I < Size; I++) + for (size_t I = 0, J = 0; I < MaskedSize; I++) if (Mask[I]) Data[I] = T[J++]; return Size; diff --git a/lib/fuzzer/FuzzerMutate.h b/lib/fuzzer/FuzzerMutate.h index a51c7fb44d46..6cbce8027624 100644 --- a/lib/fuzzer/FuzzerMutate.h +++ b/lib/fuzzer/FuzzerMutate.h @@ -1,9 +1,8 @@ //===- FuzzerMutate.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::MutationDispatcher @@ -64,7 +63,7 @@ public: /// Change a 1-, 2-, 4-, or 8-byte integer in interesting ways. size_t Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize); - /// CrossOver Data with some other element of the corpus. + /// CrossOver Data with CrossOverWith. size_t Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize); /// Applies one of the configured mutations. @@ -89,7 +88,7 @@ public: void PrintRecommendedDictionary(); - void SetCorpus(const InputCorpus *Corpus) { this->Corpus = Corpus; } + void SetCrossOverWith(const Unit *U) { CrossOverWith = U; } Random &GetRand() { return Rand; } @@ -140,7 +139,7 @@ public: DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; - const InputCorpus *Corpus = nullptr; + const Unit *CrossOverWith = nullptr; Vector<uint8_t> MutateInPlaceHere; Vector<uint8_t> MutateWithMaskTemp; // CustomCrossOver needs its own buffer as a custom implementation may call diff --git a/lib/fuzzer/FuzzerOptions.h b/lib/fuzzer/FuzzerOptions.h index ab90df82a63d..ad3df015bc77 100644 --- a/lib/fuzzer/FuzzerOptions.h +++ b/lib/fuzzer/FuzzerOptions.h @@ -1,8 +1,7 @@ // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::FuzzingOptions @@ -20,8 +19,13 @@ struct FuzzingOptions { size_t MaxLen = 0; size_t LenControl = 1000; int UnitTimeoutSec = 300; - int TimeoutExitCode = 77; + int TimeoutExitCode = 70; + int OOMExitCode = 71; + int InterruptExitCode = 72; int ErrorExitCode = 77; + bool IgnoreTimeouts = true; + bool IgnoreOOMs = true; + bool IgnoreCrashes = false; int MaxTotalTimeSec = 0; int RssLimitMb = 0; int MallocLimitMb = 0; @@ -47,6 +51,9 @@ struct FuzzingOptions { std::string ExitOnItem; std::string FocusFunction; std::string DataFlowTrace; + std::string CollectDataFlow; + std::string FeaturesDir; + std::string StopFile; bool SaveArtifacts = true; bool PrintNEW = true; // Print a status line when new units are found; bool PrintNewCovPcs = false; @@ -68,6 +75,7 @@ struct FuzzingOptions { bool HandleXfsz = false; bool HandleUsr1 = false; bool HandleUsr2 = false; + bool LazyCounters = false; }; } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerRandom.h b/lib/fuzzer/FuzzerRandom.h index 8a1aa3ef5fdc..659283eee207 100644 --- a/lib/fuzzer/FuzzerRandom.h +++ b/lib/fuzzer/FuzzerRandom.h @@ -1,9 +1,8 @@ //===- FuzzerRandom.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::Random @@ -15,12 +14,17 @@ #include <random> namespace fuzzer { -class Random : public std::mt19937 { +class Random : public std::minstd_rand { public: - Random(unsigned int seed) : std::mt19937(seed) {} - result_type operator()() { return this->std::mt19937::operator()(); } + Random(unsigned int seed) : std::minstd_rand(seed) {} + result_type operator()() { return this->std::minstd_rand::operator()(); } size_t Rand() { return this->operator()(); } size_t RandBool() { return Rand() % 2; } + size_t SkewTowardsLast(size_t n) { + size_t T = this->operator()(n * n); + size_t Res = sqrt(T); + return Res; + } size_t operator()(size_t n) { return n ? Rand() % n : 0; } intptr_t operator()(intptr_t From, intptr_t To) { assert(From < To); diff --git a/lib/fuzzer/FuzzerSHA1.cpp b/lib/fuzzer/FuzzerSHA1.cpp index d2f8e811bbf8..43e5e78cd787 100644 --- a/lib/fuzzer/FuzzerSHA1.cpp +++ b/lib/fuzzer/FuzzerSHA1.cpp @@ -1,9 +1,8 @@ //===- FuzzerSHA1.h - Private copy of the SHA1 implementation ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This code is taken from public domain @@ -32,7 +31,8 @@ namespace { // Added for LibFuzzer #ifdef __BIG_ENDIAN__ # define SHA_BIG_ENDIAN -#elif defined __LITTLE_ENDIAN__ +// Windows is always little endian and MSVC doesn't have <endian.h> +#elif defined __LITTLE_ENDIAN__ || LIBFUZZER_WINDOWS /* override */ #elif defined __BYTE_ORDER # if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ diff --git a/lib/fuzzer/FuzzerSHA1.h b/lib/fuzzer/FuzzerSHA1.h index 3b5e6e807f42..05cbacda87d0 100644 --- a/lib/fuzzer/FuzzerSHA1.h +++ b/lib/fuzzer/FuzzerSHA1.h @@ -1,9 +1,8 @@ //===- FuzzerSHA1.h - Internal header for the SHA1 utils --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // SHA1 utils. diff --git a/lib/fuzzer/FuzzerShmem.h b/lib/fuzzer/FuzzerShmem.h deleted file mode 100644 index 53568e0acb69..000000000000 --- a/lib/fuzzer/FuzzerShmem.h +++ /dev/null @@ -1,69 +0,0 @@ -//===- FuzzerShmem.h - shared memory interface ------------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// - -#ifndef LLVM_FUZZER_SHMEM_H -#define LLVM_FUZZER_SHMEM_H - -#include <algorithm> -#include <cstring> -#include <string> - -#include "FuzzerDefs.h" - -namespace fuzzer { - -class SharedMemoryRegion { - public: - bool Create(const char *Name); - bool Open(const char *Name); - bool Destroy(const char *Name); - uint8_t *GetData() { return Data; } - void PostServer() {Post(0);} - void WaitServer() {Wait(0);} - void PostClient() {Post(1);} - void WaitClient() {Wait(1);} - - size_t WriteByteArray(const uint8_t *Bytes, size_t N) { - assert(N <= kShmemSize - sizeof(N)); - memcpy(GetData(), &N, sizeof(N)); - memcpy(GetData() + sizeof(N), Bytes, N); - assert(N == ReadByteArraySize()); - return N; - } - size_t ReadByteArraySize() { - size_t Res; - memcpy(&Res, GetData(), sizeof(Res)); - return Res; - } - uint8_t *GetByteArray() { return GetData() + sizeof(size_t); } - - bool IsServer() const { return Data && IAmServer; } - bool IsClient() const { return Data && !IAmServer; } - -private: - - static const size_t kShmemSize = 1 << 22; - bool IAmServer; - std::string Path(const char *Name); - std::string SemName(const char *Name, int Idx); - void Post(int Idx); - void Wait(int Idx); - - bool Map(int fd); - uint8_t *Data = nullptr; - void *Semaphore[2]; -}; - -extern SharedMemoryRegion SMR; - -} // namespace fuzzer - -#endif // LLVM_FUZZER_SHMEM_H diff --git a/lib/fuzzer/FuzzerShmemFuchsia.cpp b/lib/fuzzer/FuzzerShmemFuchsia.cpp deleted file mode 100644 index e9ce50c2ac8a..000000000000 --- a/lib/fuzzer/FuzzerShmemFuchsia.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion. For Fuchsia, this is just stubs as equivalence servers -// are not currently supported. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" - -#if LIBFUZZER_FUCHSIA - -#include "FuzzerShmem.h" - -namespace fuzzer { - -bool SharedMemoryRegion::Create(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return false; -} - -void SharedMemoryRegion::Post(int Idx) {} - -void SharedMemoryRegion::Wait(int Idx) {} - -} // namespace fuzzer - -#endif // LIBFUZZER_FUCHSIA diff --git a/lib/fuzzer/FuzzerShmemPosix.cpp b/lib/fuzzer/FuzzerShmemPosix.cpp deleted file mode 100644 index 41a93f61004b..000000000000 --- a/lib/fuzzer/FuzzerShmemPosix.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_POSIX - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include <errno.h> -#include <fcntl.h> -#include <semaphore.h> -#include <stdio.h> -#include <stdlib.h> -#include <sys/mman.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <unistd.h> - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - // When passing a name without a leading <slash> character to - // sem_open, the behaviour is unspecified in POSIX. Add a leading - // <slash> character for the name if there is no such one. - if (!Res.empty() && Res[0] != '/') - Res.insert(Res.begin(), '/'); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - Data = - (uint8_t *)mmap(0, kShmemSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); - if (Data == (uint8_t*)-1) - return false; - return true; -} - -bool SharedMemoryRegion::Create(const char *Name) { - int fd = open(Path(Name).c_str(), O_CREAT | O_RDWR, 0777); - if (fd < 0) return false; - if (ftruncate(fd, kShmemSize) < 0) return false; - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - sem_unlink(SemName(Name, i).c_str()); - Semaphore[i] = sem_open(SemName(Name, i).c_str(), O_CREAT, 0644, 0); - if (Semaphore[i] == SEM_FAILED) - return false; - } - IAmServer = true; - return true; -} - -bool SharedMemoryRegion::Open(const char *Name) { - int fd = open(Path(Name).c_str(), O_RDWR); - if (fd < 0) return false; - struct stat stat_res; - if (0 != fstat(fd, &stat_res)) - return false; - assert(stat_res.st_size == kShmemSize); - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - Semaphore[i] = sem_open(SemName(Name, i).c_str(), 0); - if (Semaphore[i] == SEM_FAILED) - return false; - } - IAmServer = false; - return true; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return 0 == unlink(Path(Name).c_str()); -} - -void SharedMemoryRegion::Post(int Idx) { - assert(Idx == 0 || Idx == 1); - sem_post((sem_t*)Semaphore[Idx]); -} - -void SharedMemoryRegion::Wait(int Idx) { - assert(Idx == 0 || Idx == 1); - for (int i = 0; i < 10 && sem_wait((sem_t*)Semaphore[Idx]); i++) { - // sem_wait may fail if interrupted by a signal. - sleep(i); - if (i) - Printf("%s: sem_wait[%d] failed %s\n", i < 9 ? "WARNING" : "ERROR", i, - strerror(errno)); - if (i == 9) abort(); - } -} - -} // namespace fuzzer - -#endif // LIBFUZZER_POSIX diff --git a/lib/fuzzer/FuzzerShmemWindows.cpp b/lib/fuzzer/FuzzerShmemWindows.cpp deleted file mode 100644 index d330ebf4fd07..000000000000 --- a/lib/fuzzer/FuzzerShmemWindows.cpp +++ /dev/null @@ -1,64 +0,0 @@ -//===- FuzzerShmemWindows.cpp - Posix shared memory -------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include <fcntl.h> -#include <stdio.h> -#include <sys/stat.h> -#include <sys/types.h> - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Create(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -void SharedMemoryRegion::Post(int Idx) { - assert(0 && "UNIMPLEMENTED"); -} - -void SharedMemoryRegion::Wait(int Idx) { - Semaphore[1] = nullptr; - assert(0 && "UNIMPLEMENTED"); -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/lib/fuzzer/FuzzerTracePC.cpp b/lib/fuzzer/FuzzerTracePC.cpp index 80b33105bb22..4a1308de5504 100644 --- a/lib/fuzzer/FuzzerTracePC.cpp +++ b/lib/fuzzer/FuzzerTracePC.cpp @@ -1,9 +1,8 @@ //===- FuzzerTracePC.cpp - PC tracing--------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Trace PCs. @@ -24,15 +23,6 @@ #include "FuzzerValueBitMap.h" #include <set> -// The coverage counters and PCs. -// These are declared as global variables named "__sancov_*" to simplify -// experiments with inlined instrumentation. -alignas(64) ATTRIBUTE_INTERFACE -uint8_t __sancov_trace_pc_guard_8bit_counters[fuzzer::TracePC::kNumPCs]; - -ATTRIBUTE_INTERFACE -uintptr_t __sancov_trace_pc_pcs[fuzzer::TracePC::kNumPCs]; - // Used by -fsanitize-coverage=stack-depth to track stack depth ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC uintptr_t __sancov_lowest_stack; @@ -40,33 +30,80 @@ namespace fuzzer { TracePC TPC; -uint8_t *TracePC::Counters() const { - return __sancov_trace_pc_guard_8bit_counters; -} - -uintptr_t *TracePC::PCs() const { - return __sancov_trace_pc_pcs; -} - size_t TracePC::GetTotalPCCoverage() { - if (ObservedPCs.size()) - return ObservedPCs.size(); - size_t Res = 0; - for (size_t i = 1, N = GetNumPCs(); i < N; i++) - if (PCs()[i]) - Res++; - return Res; + return ObservedPCs.size(); } void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { if (Start == Stop) return; - if (NumModulesWithInline8bitCounters && - ModuleCounters[NumModulesWithInline8bitCounters-1].Start == Start) return; - assert(NumModulesWithInline8bitCounters < - sizeof(ModuleCounters) / sizeof(ModuleCounters[0])); - ModuleCounters[NumModulesWithInline8bitCounters++] = {Start, Stop}; - NumInline8bitCounters += Stop - Start; + if (NumModules && + Modules[NumModules - 1].Start() == Start) + return; + assert(NumModules < + sizeof(Modules) / sizeof(Modules[0])); + auto &M = Modules[NumModules++]; + uint8_t *AlignedStart = RoundUpByPage(Start); + uint8_t *AlignedStop = RoundDownByPage(Stop); + size_t NumFullPages = AlignedStop > AlignedStart ? + (AlignedStop - AlignedStart) / PageSize() : 0; + bool NeedFirst = Start < AlignedStart || !NumFullPages; + bool NeedLast = Stop > AlignedStop && AlignedStop >= AlignedStart; + M.NumRegions = NumFullPages + NeedFirst + NeedLast;; + assert(M.NumRegions > 0); + M.Regions = new Module::Region[M.NumRegions]; + assert(M.Regions); + size_t R = 0; + if (NeedFirst) + M.Regions[R++] = {Start, std::min(Stop, AlignedStart), true, false}; + for (uint8_t *P = AlignedStart; P < AlignedStop; P += PageSize()) + M.Regions[R++] = {P, P + PageSize(), true, true}; + if (NeedLast) + M.Regions[R++] = {AlignedStop, Stop, true, false}; + assert(R == M.NumRegions); + assert(M.Size() == (size_t)(Stop - Start)); + assert(M.Stop() == Stop); + assert(M.Start() == Start); + NumInline8bitCounters += M.Size(); +} + +// Mark all full page counter regions as PROT_NONE and set Enabled=false. +// The first time the instrumented code hits such a protected/disabled +// counter region we should catch a SEGV and call UnprotectLazyCounters, +// which will mark the page as PROT_READ|PROT_WRITE and set Enabled=true. +// +// Whenever other functions iterate over the counters they should ignore +// regions with Enabled=false. +void TracePC::ProtectLazyCounters() { + size_t NumPagesProtected = 0; + IterateCounterRegions([&](Module::Region &R) { + if (!R.OneFullPage) return; + if (Mprotect(R.Start, R.Stop - R.Start, false)) { + R.Enabled = false; + NumPagesProtected++; + } + }); + if (NumPagesProtected) + Printf("INFO: %zd pages of counters where protected;" + " libFuzzer's SEGV handler must be installed\n", + NumPagesProtected); +} + +bool TracePC::UnprotectLazyCounters(void *CounterPtr) { + // Printf("UnprotectLazyCounters: %p\n", CounterPtr); + if (!CounterPtr) + return false; + bool Done = false; + uint8_t *Addr = reinterpret_cast<uint8_t *>(CounterPtr); + IterateCounterRegions([&](Module::Region &R) { + if (!R.OneFullPage || R.Enabled || Done) return; + if (Addr >= R.Start && Addr < R.Stop) + if (Mprotect(R.Start, R.Stop - R.Start, true)) { + R.Enabled = true; + Done = true; + } + }); + return Done; } void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { @@ -78,38 +115,13 @@ void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { NumPCsInPCTables += E - B; } -void TracePC::HandleInit(uint32_t *Start, uint32_t *Stop) { - if (Start == Stop || *Start) return; - assert(NumModules < sizeof(Modules) / sizeof(Modules[0])); - for (uint32_t *P = Start; P < Stop; P++) { - NumGuards++; - if (NumGuards == kNumPCs) { - RawPrint( - "WARNING: The binary has too many instrumented PCs.\n" - " You may want to reduce the size of the binary\n" - " for more efficient fuzzing and precise coverage data\n"); - } - *P = NumGuards % kNumPCs; - } - Modules[NumModules].Start = Start; - Modules[NumModules].Stop = Stop; - NumModules++; -} - void TracePC::PrintModuleInfo() { - if (NumGuards) { - Printf("INFO: Loaded %zd modules (%zd guards): ", NumModules, NumGuards); - for (size_t i = 0; i < NumModules; i++) - Printf("%zd [%p, %p), ", Modules[i].Stop - Modules[i].Start, - Modules[i].Start, Modules[i].Stop); - Printf("\n"); - } - if (NumModulesWithInline8bitCounters) { + if (NumModules) { Printf("INFO: Loaded %zd modules (%zd inline 8-bit counters): ", - NumModulesWithInline8bitCounters, NumInline8bitCounters); - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) - Printf("%zd [%p, %p), ", ModuleCounters[i].Stop - ModuleCounters[i].Start, - ModuleCounters[i].Start, ModuleCounters[i].Stop); + NumModules, NumInline8bitCounters); + for (size_t i = 0; i < NumModules; i++) + Printf("%zd [%p, %p), ", Modules[i].Size(), Modules[i].Start(), + Modules[i].Stop()); Printf("\n"); } if (NumPCTables) { @@ -121,8 +133,7 @@ void TracePC::PrintModuleInfo() { } Printf("\n"); - if ((NumGuards && NumGuards != NumPCsInPCTables) || - (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables)) { + if (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables) { Printf("ERROR: The size of coverage PC tables does not match the\n" "number of instrumented PCs. This might be a compiler bug,\n" "please contact the libFuzzer developers.\n" @@ -163,7 +174,7 @@ inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { /// \return the address of the next instruction. /// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.cc` -inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { +ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { #if defined(__mips__) return PC + 8; #elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ @@ -176,41 +187,34 @@ inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { void TracePC::UpdateObservedPCs() { Vector<uintptr_t> CoveredFuncs; - auto ObservePC = [&](uintptr_t PC) { - if (ObservedPCs.insert(PC).second && DoPrintNewPCs) { - PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", GetNextInstructionPc(PC)); + auto ObservePC = [&](const PCTableEntry *TE) { + if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", + GetNextInstructionPc(TE->PC)); Printf("\n"); } }; - auto Observe = [&](const PCTableEntry &TE) { - if (TE.PCFlags & 1) - if (++ObservedFuncs[TE.PC] == 1 && NumPrintNewFuncs) - CoveredFuncs.push_back(TE.PC); - ObservePC(TE.PC); + auto Observe = [&](const PCTableEntry *TE) { + if (PcIsFuncEntry(TE)) + if (++ObservedFuncs[TE->PC] == 1 && NumPrintNewFuncs) + CoveredFuncs.push_back(TE->PC); + ObservePC(TE); }; if (NumPCsInPCTables) { if (NumInline8bitCounters == NumPCsInPCTables) { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - assert(Size == - (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++) - if (Beg[j]) - Observe(ModulePCTable[i].Start[j]); - } - } else if (NumGuards == NumPCsInPCTables) { - size_t GuardIdx = 1; for (size_t i = 0; i < NumModules; i++) { - uint32_t *Beg = Modules[i].Start; - size_t Size = Modules[i].Stop - Beg; - assert(Size == + auto &M = Modules[i]; + assert(M.Size() == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++, GuardIdx++) - if (Counters()[GuardIdx]) - Observe(ModulePCTable[i].Start[j]); + for (size_t r = 0; r < M.NumRegions; r++) { + auto &R = M.Regions[r]; + if (!R.Enabled) continue; + for (uint8_t *P = R.Start; P < R.Stop; P++) + if (*P) + Observe(&ModulePCTable[i].Start[M.Idx(P)]); + } } } } @@ -223,6 +227,27 @@ void TracePC::UpdateObservedPCs() { } } +uintptr_t TracePC::PCTableEntryIdx(const PCTableEntry *TE) { + size_t TotalTEs = 0; + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + if (TE >= M.Start && TE < M.Stop) + return TotalTEs + TE - M.Start; + TotalTEs += M.Stop - M.Start; + } + assert(0); + return 0; +} + +const TracePC::PCTableEntry *TracePC::PCTableEntryByIdx(uintptr_t Idx) { + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + size_t Size = M.Stop - M.Start; + if (Idx < Size) return &M.Start[Idx]; + Idx -= Size; + } + return nullptr; +} static std::string GetModuleName(uintptr_t PC) { char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? @@ -242,47 +267,38 @@ void TracePC::IterateCoveredFunctions(CallBack CB) { auto ModuleName = GetModuleName(M.Start->PC); for (auto NextFE = M.Start; NextFE < M.Stop; ) { auto FE = NextFE; - assert((FE->PCFlags & 1) && "Not a function entry point"); + assert(PcIsFuncEntry(FE) && "Not a function entry point"); do { NextFE++; - } while (NextFE < M.Stop && !(NextFE->PCFlags & 1)); - if (ObservedFuncs.count(FE->PC)) - CB(FE, NextFE, ObservedFuncs[FE->PC]); + } while (NextFE < M.Stop && !(PcIsFuncEntry(NextFE))); + CB(FE, NextFE, ObservedFuncs[FE->PC]); } } } void TracePC::SetFocusFunction(const std::string &FuncName) { // This function should be called once. - assert(FocusFunction.first > NumModulesWithInline8bitCounters); + assert(!FocusFunctionCounterPtr); if (FuncName.empty()) return; - for (size_t M = 0; M < NumModulesWithInline8bitCounters; M++) { + for (size_t M = 0; M < NumModules; M++) { auto &PCTE = ModulePCTable[M]; size_t N = PCTE.Stop - PCTE.Start; for (size_t I = 0; I < N; I++) { - if (!(PCTE.Start[I].PCFlags & 1)) continue; // not a function entry. + if (!(PcIsFuncEntry(&PCTE.Start[I]))) continue; // not a function entry. auto Name = DescribePC("%F", GetNextInstructionPc(PCTE.Start[I].PC)); if (Name[0] == 'i' && Name[1] == 'n' && Name[2] == ' ') Name = Name.substr(3, std::string::npos); if (FuncName != Name) continue; Printf("INFO: Focus function is set to '%s'\n", Name.c_str()); - FocusFunction = {M, I}; + FocusFunctionCounterPtr = Modules[M].Start() + I; return; } } } bool TracePC::ObservedFocusFunction() { - size_t I = FocusFunction.first; - size_t J = FocusFunction.second; - if (I >= NumModulesWithInline8bitCounters) - return false; - auto &MC = ModuleCounters[I]; - size_t Size = MC.Stop - MC.Start; - if (J >= Size) - return false; - return MC.Start[J] != 0; + return FocusFunctionCounterPtr && *FocusFunctionCounterPtr; } void TracePC::PrintCoverage() { @@ -306,32 +322,24 @@ void TracePC::PrintCoverage() { if (FunctionStr.find("in ") == 0) FunctionStr = FunctionStr.substr(3); std::string LineStr = DescribePC("%l", VisualizePC); - size_t Line = std::stoul(LineStr); size_t NumEdges = Last - First; Vector<uintptr_t> UncoveredPCs; for (auto TE = First; TE < Last; TE++) - if (!ObservedPCs.count(TE->PC)) + if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); - Printf("COVERED_FUNC: hits: %zd", Counter); + Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); - Printf(" %s %s:%zd\n", FunctionStr.c_str(), FileStr.c_str(), Line); - for (auto PC: UncoveredPCs) - Printf(" UNCOVERED_PC: %s\n", - DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); + Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(), + LineStr.c_str()); + if (Counter) + for (auto PC : UncoveredPCs) + Printf(" UNCOVERED_PC: %s\n", + DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); }; IterateCoveredFunctions(CoveredFunctionCallback); } -void TracePC::DumpCoverage() { - if (EF->__sanitizer_dump_coverage) { - Vector<uintptr_t> PCsCopy(GetNumPCs()); - for (size_t i = 0; i < GetNumPCs(); i++) - PCsCopy[i] = PCs()[i] ? GetPreviousInstructionPc(PCs()[i]) : 0; - EF->__sanitizer_dump_coverage(PCsCopy.data(), PCsCopy.size()); - } -} - // Value profile. // We keep track of various values that affect control flow. // These values are inserted into a bit-set-based hash map. @@ -361,11 +369,16 @@ void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, Hash ^= (T << 8) | B2[i]; } size_t I = 0; - for (; I < Len; I++) - if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) + uint8_t HammingDistance = 0; + for (; I < Len; I++) { + if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) { + HammingDistance = Popcountll(B1[I] ^ B2[I]); break; + } + } size_t PC = reinterpret_cast<size_t>(caller_pc); size_t Idx = (PC & 4095) | (I << 12); + Idx += HammingDistance; ValueProfileMap.AddValue(Idx); TORCW.Insert(Idx ^ Hash, Word(B1, Len), Word(B2, Len)); } @@ -400,11 +413,10 @@ static size_t InternalStrnlen2(const char *S1, const char *S2) { } void TracePC::ClearInlineCounters() { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - memset(Beg, 0, Size); - } + IterateCounterRegions([](const Module::Region &R){ + if (R.Enabled) + memset(R.Start, 0, R.Stop - R.Start); + }); } ATTRIBUTE_NO_SANITIZE_ALL @@ -417,16 +429,25 @@ uintptr_t TracePC::GetMaxStackOffset() const { return InitialStack - __sancov_lowest_stack; // Stack grows down } +void WarnAboutDeprecatedInstrumentation(const char *flag) { + // Use RawPrint because Printf cannot be used on Windows before OutputFile is + // initialized. + RawPrint(flag); + RawPrint( + " is no longer supported by libFuzzer.\n" + "Please either migrate to a compiler that supports -fsanitize=fuzzer\n" + "or use an older version of libFuzzer\n"); + exit(1); +} + } // namespace fuzzer extern "C" { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { - uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); - uint32_t Idx = *Guard; - __sancov_trace_pc_pcs[Idx] = PC; - __sancov_trace_pc_guard_8bit_counters[Idx]++; + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); } // Best-effort support for -fsanitize-coverage=trace-pc, which is available @@ -434,15 +455,13 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc() { - uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); - uintptr_t Idx = PC & (((uintptr_t)1 << fuzzer::TracePC::kTracePcBits) - 1); - __sancov_trace_pc_pcs[Idx] = PC; - __sancov_trace_pc_guard_8bit_counters[Idx]++; + fuzzer::WarnAboutDeprecatedInstrumentation("-fsanitize-coverage=trace-pc"); } ATTRIBUTE_INTERFACE void __sanitizer_cov_trace_pc_guard_init(uint32_t *Start, uint32_t *Stop) { - fuzzer::TPC.HandleInit(Start, Stop); + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); } ATTRIBUTE_INTERFACE @@ -537,24 +556,44 @@ void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { uint64_t N = Cases[0]; uint64_t ValSizeInBits = Cases[1]; uint64_t *Vals = Cases + 2; - // Skip the most common and the most boring case. - if (Vals[N - 1] < 256 && Val < 256) + // Skip the most common and the most boring case: all switch values are small. + // We may want to skip this at compile-time, but it will make the + // instrumentation less general. + if (Vals[N - 1] < 256) + return; + // Also skip small inputs values, they won't give good signal. + if (Val < 256) return; uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); size_t i; - uint64_t Token = 0; + uint64_t Smaller = 0; + uint64_t Larger = ~(uint64_t)0; + // Find two switch values such that Smaller < Val < Larger. + // Use 0 and 0xfff..f as the defaults. for (i = 0; i < N; i++) { - Token = Val ^ Vals[i]; - if (Val < Vals[i]) + if (Val < Vals[i]) { + Larger = Vals[i]; break; + } + if (Val > Vals[i]) Smaller = Vals[i]; } - if (ValSizeInBits == 16) - fuzzer::TPC.HandleCmp(PC + i, static_cast<uint16_t>(Token), (uint16_t)(0)); - else if (ValSizeInBits == 32) - fuzzer::TPC.HandleCmp(PC + i, static_cast<uint32_t>(Token), (uint32_t)(0)); - else - fuzzer::TPC.HandleCmp(PC + i, Token, (uint64_t)(0)); + // Apply HandleCmp to {Val,Smaller} and {Val, Larger}, + // use i as the PC modifier for HandleCmp. + if (ValSizeInBits == 16) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast<uint16_t>(Val), + (uint16_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast<uint16_t>(Val), + (uint16_t)(Larger)); + } else if (ValSizeInBits == 32) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast<uint32_t>(Val), + (uint32_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast<uint32_t>(Val), + (uint32_t)(Larger)); + } else { + fuzzer::TPC.HandleCmp(PC + 2*i, Val, Smaller); + fuzzer::TPC.HandleCmp(PC + 2*i + 1, Val, Larger); + } } ATTRIBUTE_INTERFACE diff --git a/lib/fuzzer/FuzzerTracePC.h b/lib/fuzzer/FuzzerTracePC.h index 46d6c24887fd..4f5ebeb047a1 100644 --- a/lib/fuzzer/FuzzerTracePC.h +++ b/lib/fuzzer/FuzzerTracePC.h @@ -1,9 +1,8 @@ //===- FuzzerTracePC.h - Internal header for the Fuzzer ---------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::TracePC @@ -70,11 +69,6 @@ struct MemMemTable { class TracePC { public: - static const size_t kNumPCs = 1 << 21; - // How many bits of PC are used from __sanitizer_cov_trace_pc. - static const size_t kTracePcBits = 18; - - void HandleInit(uint32_t *Start, uint32_t *Stop); void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); void HandleCallerCallee(uintptr_t Caller, uintptr_t Callee); @@ -89,8 +83,6 @@ class TracePC { void ResetMaps() { ValueProfileMap.Reset(); - if (NumModules) - memset(Counters(), 0, GetNumPCs()); ClearExtraCounters(); ClearInlineCounters(); } @@ -103,7 +95,6 @@ class TracePC { void PrintModuleInfo(); void PrintCoverage(); - void DumpCoverage(); template<class CallBack> void IterateCoveredFunctions(CallBack CB); @@ -116,14 +107,6 @@ class TracePC { TableOfRecentCompares<Word, 32> TORCW; MemMemTable<1024> MMT; - size_t GetNumPCs() const { - return NumGuards == 0 ? (1 << kTracePcBits) : Min(kNumPCs, NumGuards + 1); - } - uintptr_t GetPC(size_t Idx) { - assert(Idx < GetNumPCs()); - return PCs()[Idx]; - } - void RecordInitialStack(); uintptr_t GetMaxStackOffset() const; @@ -136,39 +119,63 @@ class TracePC { void SetFocusFunction(const std::string &FuncName); bool ObservedFocusFunction(); + void ProtectLazyCounters(); + bool UnprotectLazyCounters(void *CounterPtr); + + struct PCTableEntry { + uintptr_t PC, PCFlags; + }; + + uintptr_t PCTableEntryIdx(const PCTableEntry *TE); + const PCTableEntry *PCTableEntryByIdx(uintptr_t Idx); + static uintptr_t GetNextInstructionPc(uintptr_t PC); + bool PcIsFuncEntry(const PCTableEntry *TE) { return TE->PCFlags & 1; } + private: bool UseCounters = false; uint32_t UseValueProfileMask = false; bool DoPrintNewPCs = false; size_t NumPrintNewFuncs = 0; + // Module represents the array of 8-bit counters split into regions + // such that every region, except maybe the first and the last one, is one + // full page. struct Module { - uint32_t *Start, *Stop; + struct Region { + uint8_t *Start, *Stop; + bool Enabled; + bool OneFullPage; + }; + Region *Regions; + size_t NumRegions; + uint8_t *Start() { return Regions[0].Start; } + uint8_t *Stop() { return Regions[NumRegions - 1].Stop; } + size_t Size() { return Stop() - Start(); } + size_t Idx(uint8_t *P) { + assert(P >= Start() && P < Stop()); + return P - Start(); + } }; Module Modules[4096]; size_t NumModules; // linker-initialized. - size_t NumGuards; // linker-initialized. - - struct { uint8_t *Start, *Stop; } ModuleCounters[4096]; - size_t NumModulesWithInline8bitCounters; // linker-initialized. size_t NumInline8bitCounters; - struct PCTableEntry { - uintptr_t PC, PCFlags; - }; + template <class Callback> + void IterateCounterRegions(Callback CB) { + for (size_t m = 0; m < NumModules; m++) + for (size_t r = 0; r < Modules[m].NumRegions; r++) + CB(Modules[m].Regions[r]); + } struct { const PCTableEntry *Start, *Stop; } ModulePCTable[4096]; size_t NumPCTables; size_t NumPCsInPCTables; - uint8_t *Counters() const; - uintptr_t *PCs() const; - - Set<uintptr_t> ObservedPCs; + Set<const PCTableEntry*> ObservedPCs; std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter. - std::pair<size_t, size_t> FocusFunction = {-1, -1}; // Module and PC IDs. + uint8_t *FocusFunctionCounterPtr = nullptr; ValueBitMap ValueProfileMap; uintptr_t InitialStack; @@ -177,7 +184,7 @@ private: template <class Callback> // void Callback(size_t FirstFeature, size_t Idx, uint8_t Value); ATTRIBUTE_NO_SANITIZE_ALL -void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, +size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, size_t FirstFeature, Callback Handle8bitCounter) { typedef uintptr_t LargeType; const size_t Step = sizeof(LargeType) / sizeof(uint8_t); @@ -199,6 +206,7 @@ void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, for (; P < End; P++) if (uint8_t V = *P) Handle8bitCounter(FirstFeature, P - Begin, V); + return End - Begin; } // Given a non-zero Counter returns a number in the range [0,7]. @@ -229,10 +237,8 @@ unsigned CounterToFeature(T Counter) { template <class Callback> // void Callback(size_t Feature) ATTRIBUTE_NO_SANITIZE_ADDRESS -__attribute__((noinline)) +ATTRIBUTE_NOINLINE void TracePC::CollectFeatures(Callback HandleFeature) const { - uint8_t *Counters = this->Counters(); - size_t N = GetNumPCs(); auto Handle8bitCounter = [&](size_t FirstFeature, size_t Idx, uint8_t Counter) { if (UseCounters) @@ -243,22 +249,18 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { size_t FirstFeature = 0; - if (!NumInline8bitCounters) { - ForEachNonZeroByte(Counters, Counters + N, FirstFeature, Handle8bitCounter); - FirstFeature += N * 8; - } - - if (NumInline8bitCounters) { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - ForEachNonZeroByte(ModuleCounters[i].Start, ModuleCounters[i].Stop, - FirstFeature, Handle8bitCounter); - FirstFeature += 8 * (ModuleCounters[i].Stop - ModuleCounters[i].Start); + for (size_t i = 0; i < NumModules; i++) { + for (size_t r = 0; r < Modules[i].NumRegions; r++) { + if (!Modules[i].Regions[r].Enabled) continue; + FirstFeature += 8 * ForEachNonZeroByte(Modules[i].Regions[r].Start, + Modules[i].Regions[r].Stop, + FirstFeature, Handle8bitCounter); } } - ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), FirstFeature, - Handle8bitCounter); - FirstFeature += (ExtraCountersEnd() - ExtraCountersBegin()) * 8; + FirstFeature += + 8 * ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), + FirstFeature, Handle8bitCounter); if (UseValueProfileMask) { ValueProfileMap.ForEach([&](size_t Idx) { diff --git a/lib/fuzzer/FuzzerUtil.cpp b/lib/fuzzer/FuzzerUtil.cpp index 6286f9a718ad..7aa84a1faad7 100644 --- a/lib/fuzzer/FuzzerUtil.cpp +++ b/lib/fuzzer/FuzzerUtil.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtil.cpp - Misc utils ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils. diff --git a/lib/fuzzer/FuzzerUtil.h b/lib/fuzzer/FuzzerUtil.h index d2f1d5de4267..0a127911df3c 100644 --- a/lib/fuzzer/FuzzerUtil.h +++ b/lib/fuzzer/FuzzerUtil.h @@ -1,9 +1,8 @@ //===- FuzzerUtil.h - Internal header for the Fuzzer Utils ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Util functions. @@ -53,6 +52,8 @@ void SetSignalHandler(const FuzzingOptions& Options); void SleepSeconds(int Seconds); +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite); + unsigned long GetPid(); size_t GetPeakRSSMb(); @@ -88,6 +89,20 @@ size_t SimpleFastHash(const uint8_t *Data, size_t Size); inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } +inline size_t PageSize() { return 4096; } +inline uint8_t *RoundUpByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast<uintptr_t>(P); + size_t Mask = PageSize() - 1; + X = (X + Mask) & ~Mask; + return reinterpret_cast<uint8_t *>(X); +} +inline uint8_t *RoundDownByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast<uintptr_t>(P); + size_t Mask = PageSize() - 1; + X = X & ~Mask; + return reinterpret_cast<uint8_t *>(X); +} + } // namespace fuzzer #endif // LLVM_FUZZER_UTIL_H diff --git a/lib/fuzzer/FuzzerUtilDarwin.cpp b/lib/fuzzer/FuzzerUtilDarwin.cpp index 4bfbc11a50cf..171db23570c4 100644 --- a/lib/fuzzer/FuzzerUtilDarwin.cpp +++ b/lib/fuzzer/FuzzerUtilDarwin.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilDarwin.cpp - Misc utils ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils for Darwin. diff --git a/lib/fuzzer/FuzzerUtilFuchsia.cpp b/lib/fuzzer/FuzzerUtilFuchsia.cpp index cd48fefef352..1f04b33c154e 100644 --- a/lib/fuzzer/FuzzerUtilFuchsia.cpp +++ b/lib/fuzzer/FuzzerUtilFuchsia.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilFuchsia.cpp - Misc utils for Fuchsia. --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation using Fuchsia/Zircon APIs. @@ -14,6 +13,7 @@ #include "FuzzerInternal.h" #include "FuzzerUtil.h" +#include <cassert> #include <cerrno> #include <cinttypes> #include <cstdint> @@ -30,7 +30,7 @@ #include <zircon/syscalls.h> #include <zircon/syscalls/debug.h> #include <zircon/syscalls/exception.h> -#include <zircon/syscalls/port.h> +#include <zircon/syscalls/object.h> #include <zircon/types.h> namespace fuzzer { @@ -49,10 +49,6 @@ void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); namespace { -// A magic value for the Zircon exception port, chosen to spell 'FUZZING' -// when interpreted as a byte sequence on little-endian platforms. -const uint64_t kFuzzingCrash = 0x474e495a5a5546; - // Helper function to handle Zircon syscall failures. void ExitOnErr(zx_status_t Status, const char *Syscall) { if (Status != ZX_OK) { @@ -218,76 +214,101 @@ void CrashHandler(zx_handle_t *Event) { zx_handle_t Handle = ZX_HANDLE_INVALID; }; - // Create and bind the exception port. We need to claim to be a "debugger" so - // the kernel will allow us to modify and resume dying threads (see below). - // Once the port is set, we can signal the main thread to continue and wait + // Create the exception channel. We need to claim to be a "debugger" so the + // kernel will allow us to modify and resume dying threads (see below). Once + // the channel is set, we can signal the main thread to continue and wait // for the exception to arrive. - ScopedHandle Port; - ExitOnErr(_zx_port_create(0, &Port.Handle), "_zx_port_create"); + ScopedHandle Channel; zx_handle_t Self = _zx_process_self(); - - ExitOnErr(_zx_task_bind_exception_port(Self, Port.Handle, kFuzzingCrash, - ZX_EXCEPTION_PORT_DEBUGGER), - "_zx_task_bind_exception_port"); + ExitOnErr(_zx_task_create_exception_channel( + Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle), + "_zx_task_create_exception_channel"); ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0), "_zx_object_signal"); - zx_port_packet_t Packet; - ExitOnErr(_zx_port_wait(Port.Handle, ZX_TIME_INFINITE, &Packet), - "_zx_port_wait"); - - // At this point, we want to get the state of the crashing thread, but - // libFuzzer and the sanitizers assume this will happen from that same thread - // via a POSIX signal handler. "Resurrecting" the thread in the middle of the - // appropriate callback is as simple as forcibly setting the instruction - // pointer/program counter, provided we NEVER EVER return from that function - // (since otherwise our stack will not be valid). - ScopedHandle Thread; - ExitOnErr(_zx_object_get_child(Self, Packet.exception.tid, - ZX_RIGHT_SAME_RIGHTS, &Thread.Handle), - "_zx_object_get_child"); - - zx_thread_state_general_regs_t GeneralRegisters; - ExitOnErr(_zx_thread_read_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, - &GeneralRegisters, sizeof(GeneralRegisters)), - "_zx_thread_read_state"); - - // To unwind properly, we need to push the crashing thread's register state - // onto the stack and jump into a trampoline with CFI instructions on how - // to restore it. + // This thread lives as long as the process in order to keep handling + // crashes. In practice, the first crashed thread to reach the end of the + // StaticCrashHandler will end the process. + while (true) { + ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE, + ZX_TIME_INFINITE, nullptr), + "_zx_object_wait_one"); + + zx_exception_info_t ExceptionInfo; + ScopedHandle Exception; + ExitOnErr(_zx_channel_read(Channel.Handle, 0, &ExceptionInfo, + &Exception.Handle, sizeof(ExceptionInfo), 1, + nullptr, nullptr), + "_zx_channel_read"); + + // Ignore informational synthetic exceptions. + if (ZX_EXCP_THREAD_STARTING == ExceptionInfo.type || + ZX_EXCP_THREAD_EXITING == ExceptionInfo.type || + ZX_EXCP_PROCESS_STARTING == ExceptionInfo.type) { + continue; + } + + // At this point, we want to get the state of the crashing thread, but + // libFuzzer and the sanitizers assume this will happen from that same + // thread via a POSIX signal handler. "Resurrecting" the thread in the + // middle of the appropriate callback is as simple as forcibly setting the + // instruction pointer/program counter, provided we NEVER EVER return from + // that function (since otherwise our stack will not be valid). + ScopedHandle Thread; + ExitOnErr(_zx_exception_get_thread(Exception.Handle, &Thread.Handle), + "_zx_exception_get_thread"); + + zx_thread_state_general_regs_t GeneralRegisters; + ExitOnErr(_zx_thread_read_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, + sizeof(GeneralRegisters)), + "_zx_thread_read_state"); + + // To unwind properly, we need to push the crashing thread's register state + // onto the stack and jump into a trampoline with CFI instructions on how + // to restore it. #if defined(__x86_64__) - uintptr_t StackPtr = - (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) & - -(uintptr_t)16; - __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, - sizeof(GeneralRegisters)); - GeneralRegisters.rsp = StackPtr; - GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); + uintptr_t StackPtr = + (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) & + -(uintptr_t)16; + __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.rsp = StackPtr; + GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); #elif defined(__aarch64__) - uintptr_t StackPtr = - (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16; - __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, - sizeof(GeneralRegisters)); - GeneralRegisters.sp = StackPtr; - GeneralRegisters.pc = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); + uintptr_t StackPtr = + (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16; + __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.sp = StackPtr; + GeneralRegisters.pc = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); #else #error "Unsupported architecture for fuzzing on Fuchsia" #endif - // Now force the crashing thread's state. - ExitOnErr(_zx_thread_write_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, - &GeneralRegisters, sizeof(GeneralRegisters)), - "_zx_thread_write_state"); - - ExitOnErr(_zx_task_resume_from_exception(Thread.Handle, Port.Handle, 0), - "_zx_task_resume_from_exception"); + // Now force the crashing thread's state. + ExitOnErr( + _zx_thread_write_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, sizeof(GeneralRegisters)), + "_zx_thread_write_state"); + + // Set the exception to HANDLED so it resumes the thread on close. + uint32_t ExceptionState = ZX_EXCEPTION_STATE_HANDLED; + ExitOnErr(_zx_object_set_property(Exception.Handle, ZX_PROP_EXCEPTION_STATE, + &ExceptionState, sizeof(ExceptionState)), + "zx_object_set_property"); + } } } // namespace +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return false; // UNIMPLEMENTED +} + // Platform specific functions. void SetSignalHandler(const FuzzingOptions &Options) { // Set up alarm handler if needed. diff --git a/lib/fuzzer/FuzzerUtilLinux.cpp b/lib/fuzzer/FuzzerUtilLinux.cpp index c103fd230b0c..d5a15d19f2a9 100644 --- a/lib/fuzzer/FuzzerUtilLinux.cpp +++ b/lib/fuzzer/FuzzerUtilLinux.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilLinux.cpp - Misc utils for Linux. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils for Linux. @@ -14,12 +13,18 @@ #include "FuzzerCommand.h" #include <stdlib.h> +#include <sys/types.h> +#include <sys/wait.h> + namespace fuzzer { int ExecuteCommand(const Command &Cmd) { std::string CmdLine = Cmd.toString(); - return system(CmdLine.c_str()); + int exit_code = system(CmdLine.c_str()); + if (WIFEXITED(exit_code)) + return WEXITSTATUS(exit_code); + return exit_code; } } // namespace fuzzer diff --git a/lib/fuzzer/FuzzerUtilPosix.cpp b/lib/fuzzer/FuzzerUtilPosix.cpp index bc64d3293702..110785d87413 100644 --- a/lib/fuzzer/FuzzerUtilPosix.cpp +++ b/lib/fuzzer/FuzzerUtilPosix.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilPosix.cpp - Misc utils for Posix. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation using Posix API. @@ -12,6 +11,7 @@ #if LIBFUZZER_POSIX #include "FuzzerIO.h" #include "FuzzerInternal.h" +#include "FuzzerTracePC.h" #include <cassert> #include <chrono> #include <cstring> @@ -19,6 +19,7 @@ #include <iomanip> #include <signal.h> #include <stdio.h> +#include <sys/mman.h> #include <sys/resource.h> #include <sys/syscall.h> #include <sys/time.h> @@ -32,6 +33,16 @@ static void AlarmHandler(int, siginfo_t *, void *) { Fuzzer::StaticAlarmCallback(); } +static void (*upstream_segv_handler)(int, siginfo_t *, void *); + +static void SegvHandler(int sig, siginfo_t *si, void *ucontext) { + assert(si->si_signo == SIGSEGV); + if (TPC.UnprotectLazyCounters(si->si_addr)) return; + if (upstream_segv_handler) + return upstream_segv_handler(sig, si, ucontext); + Fuzzer::StaticCrashSignalCallback(); +} + static void CrashHandler(int, siginfo_t *, void *) { Fuzzer::StaticCrashSignalCallback(); } @@ -56,8 +67,11 @@ static void SetSigaction(int signum, exit(1); } if (sigact.sa_flags & SA_SIGINFO) { - if (sigact.sa_sigaction) - return; + if (sigact.sa_sigaction) { + if (signum != SIGSEGV) + return; + upstream_segv_handler = sigact.sa_sigaction; + } } else { if (sigact.sa_handler != SIG_DFL && sigact.sa_handler != SIG_IGN && sigact.sa_handler != SIG_ERR) @@ -65,6 +79,7 @@ static void SetSigaction(int signum, } sigact = {}; + sigact.sa_flags = SA_SIGINFO; sigact.sa_sigaction = callback; if (sigaction(signum, &sigact, 0)) { Printf("libFuzzer: sigaction failed with %d\n", errno); @@ -83,6 +98,11 @@ void SetTimer(int Seconds) { SetSigaction(SIGALRM, AlarmHandler); } +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return 0 == mprotect(Ptr, Size, + AllowReadWrite ? (PROT_READ | PROT_WRITE) : PROT_NONE); +} + void SetSignalHandler(const FuzzingOptions& Options) { if (Options.UnitTimeoutSec > 0) SetTimer(Options.UnitTimeoutSec / 2 + 1); @@ -91,7 +111,7 @@ void SetSignalHandler(const FuzzingOptions& Options) { if (Options.HandleTerm) SetSigaction(SIGTERM, InterruptHandler); if (Options.HandleSegv) - SetSigaction(SIGSEGV, CrashHandler); + SetSigaction(SIGSEGV, SegvHandler); if (Options.HandleBus) SetSigaction(SIGBUS, CrashHandler); if (Options.HandleAbrt) diff --git a/lib/fuzzer/FuzzerUtilWindows.cpp b/lib/fuzzer/FuzzerUtilWindows.cpp index 393b4768be7e..074e1eb42309 100644 --- a/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/lib/fuzzer/FuzzerUtilWindows.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilWindows.cpp - Misc utils for Windows. --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation for Windows. @@ -86,11 +85,11 @@ void CALLBACK AlarmHandler(PVOID, BOOLEAN) { class TimerQ { HANDLE TimerQueue; public: - TimerQ() : TimerQueue(NULL) {}; + TimerQ() : TimerQueue(NULL) {} ~TimerQ() { if (TimerQueue) DeleteTimerQueueEx(TimerQueue, NULL); - }; + } void SetTimer(int Seconds) { if (!TimerQueue) { TimerQueue = CreateTimerQueue(); @@ -105,13 +104,17 @@ class TimerQ { Printf("libFuzzer: CreateTimerQueueTimer failed.\n"); exit(1); } - }; + } }; static TimerQ Timer; static void CrashHandler(int) { Fuzzer::StaticCrashSignalCallback(); } +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return false; // UNIMPLEMENTED +} + void SetSignalHandler(const FuzzingOptions& Options) { HandlerOpt = &Options; diff --git a/lib/fuzzer/FuzzerValueBitMap.h b/lib/fuzzer/FuzzerValueBitMap.h index 13d7cbd95dd7..bc039f1df27f 100644 --- a/lib/fuzzer/FuzzerValueBitMap.h +++ b/lib/fuzzer/FuzzerValueBitMap.h @@ -1,9 +1,8 @@ //===- FuzzerValueBitMap.h - INTERNAL - Bit map -----------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // ValueBitMap. @@ -35,7 +34,7 @@ struct ValueBitMap { uintptr_t WordIdx = Idx / kBitsInWord; uintptr_t BitIdx = Idx % kBitsInWord; uintptr_t Old = Map[WordIdx]; - uintptr_t New = Old | (1UL << BitIdx); + uintptr_t New = Old | (1ULL << BitIdx); Map[WordIdx] = New; return New != Old; } @@ -49,7 +48,7 @@ struct ValueBitMap { assert(Idx < kMapSizeInBits); uintptr_t WordIdx = Idx / kBitsInWord; uintptr_t BitIdx = Idx % kBitsInWord; - return Map[WordIdx] & (1UL << BitIdx); + return Map[WordIdx] & (1ULL << BitIdx); } size_t SizeInBits() const { return kMapSizeInBits; } @@ -65,7 +64,7 @@ struct ValueBitMap { } private: - uintptr_t Map[kMapSizeInWords] __attribute__((aligned(512))); + ATTRIBUTE_ALIGNED(512) uintptr_t Map[kMapSizeInWords]; }; } // namespace fuzzer diff --git a/lib/fuzzer/utils/FuzzedDataProvider.h b/lib/fuzzer/utils/FuzzedDataProvider.h new file mode 100644 index 000000000000..1b5b4bb01269 --- /dev/null +++ b/lib/fuzzer/utils/FuzzedDataProvider.h @@ -0,0 +1,245 @@ +//===- FuzzedDataProvider.h - Utility header for fuzz targets ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// A single header library providing an utility class to break up an array of +// bytes. Whenever run on the same input, provides the same output, as long as +// its methods are called in the same order, with the same arguments. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ +#define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ + +#include <limits.h> +#include <stddef.h> +#include <stdint.h> + +#include <algorithm> +#include <cstring> +#include <initializer_list> +#include <string> +#include <type_traits> +#include <utility> +#include <vector> + +class FuzzedDataProvider { +public: + // |data| is an array of length |size| that the FuzzedDataProvider wraps to + // provide more granular access. |data| must outlive the FuzzedDataProvider. + FuzzedDataProvider(const uint8_t *data, size_t size) + : data_ptr_(data), remaining_bytes_(size) {} + ~FuzzedDataProvider() = default; + + // Returns a std::vector containing |num_bytes| of input data. If fewer than + // |num_bytes| of data remain, returns a shorter std::vector containing all + // of the data that's left. Can be used with any byte sized type, such as + // char, unsigned char, uint8_t, etc. + template <typename T> std::vector<T> ConsumeBytes(size_t num_bytes) { + num_bytes = std::min(num_bytes, remaining_bytes_); + return ConsumeBytes<T>(num_bytes, num_bytes); + } + + // Similar to |ConsumeBytes|, but also appends the terminator value at the end + // of the resulting vector. Useful, when a mutable null-terminated C-string is + // needed, for example. But that is a rare case. Better avoid it, if possible, + // and prefer using |ConsumeBytes| or |ConsumeBytesAsString| methods. + template <typename T> + std::vector<T> ConsumeBytesWithTerminator(size_t num_bytes, + T terminator = 0) { + num_bytes = std::min(num_bytes, remaining_bytes_); + std::vector<T> result = ConsumeBytes<T>(num_bytes + 1, num_bytes); + result.back() = terminator; + return result; + } + + // Returns a std::string containing |num_bytes| of input data. Using this and + // |.c_str()| on the resulting string is the best way to get an immutable + // null-terminated C string. If fewer than |num_bytes| of data remain, returns + // a shorter std::string containing all of the data that's left. + std::string ConsumeBytesAsString(size_t num_bytes) { + static_assert(sizeof(std::string::value_type) == sizeof(uint8_t), + "ConsumeBytesAsString cannot convert the data to a string."); + + num_bytes = std::min(num_bytes, remaining_bytes_); + std::string result( + reinterpret_cast<const std::string::value_type *>(data_ptr_), + num_bytes); + Advance(num_bytes); + return result; + } + + // Returns a number in the range [min, max] by consuming bytes from the + // input data. The value might not be uniformly distributed in the given + // range. If there's no input data left, always returns |min|. |min| must + // be less than or equal to |max|. + template <typename T> T ConsumeIntegralInRange(T min, T max) { + static_assert(std::is_integral<T>::value, "An integral type is required."); + static_assert(sizeof(T) <= sizeof(uint64_t), "Unsupported integral type."); + + if (min > max) + abort(); + + // Use the biggest type possible to hold the range and the result. + uint64_t range = static_cast<uint64_t>(max) - min; + uint64_t result = 0; + size_t offset = 0; + + while (offset < sizeof(T) * CHAR_BIT && (range >> offset) > 0 && + remaining_bytes_ != 0) { + // Pull bytes off the end of the seed data. Experimentally, this seems to + // allow the fuzzer to more easily explore the input space. This makes + // sense, since it works by modifying inputs that caused new code to run, + // and this data is often used to encode length of data read by + // |ConsumeBytes|. Separating out read lengths makes it easier modify the + // contents of the data that is actually read. + --remaining_bytes_; + result = (result << CHAR_BIT) | data_ptr_[remaining_bytes_]; + offset += CHAR_BIT; + } + + // Avoid division by 0, in case |range + 1| results in overflow. + if (range != std::numeric_limits<decltype(range)>::max()) + result = result % (range + 1); + + return static_cast<T>(min + result); + } + + // Returns a std::string of length from 0 to |max_length|. When it runs out of + // input data, returns what remains of the input. Designed to be more stable + // with respect to a fuzzer inserting characters than just picking a random + // length and then consuming that many bytes with |ConsumeBytes|. + std::string ConsumeRandomLengthString(size_t max_length) { + // Reads bytes from the start of |data_ptr_|. Maps "\\" to "\", and maps "\" + // followed by anything else to the end of the string. As a result of this + // logic, a fuzzer can insert characters into the string, and the string + // will be lengthened to include those new characters, resulting in a more + // stable fuzzer than picking the length of a string independently from + // picking its contents. + std::string result; + + // Reserve the anticipated capaticity to prevent several reallocations. + result.reserve(std::min(max_length, remaining_bytes_)); + for (size_t i = 0; i < max_length && remaining_bytes_ != 0; ++i) { + char next = ConvertUnsignedToSigned<char>(data_ptr_[0]); + Advance(1); + if (next == '\\' && remaining_bytes_ != 0) { + next = ConvertUnsignedToSigned<char>(data_ptr_[0]); + Advance(1); + if (next != '\\') + break; + } + result += next; + } + + result.shrink_to_fit(); + return result; + } + + // Returns a std::vector containing all remaining bytes of the input data. + template <typename T> std::vector<T> ConsumeRemainingBytes() { + return ConsumeBytes<T>(remaining_bytes_); + } + + // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string + // object. + // Returns a std::vector containing all remaining bytes of the input data. + std::string ConsumeRemainingBytesAsString() { + return ConsumeBytesAsString(remaining_bytes_); + } + + // Returns a number in the range [Type's min, Type's max]. The value might + // not be uniformly distributed in the given range. If there's no input data + // left, always returns |min|. + template <typename T> T ConsumeIntegral() { + return ConsumeIntegralInRange(std::numeric_limits<T>::min(), + std::numeric_limits<T>::max()); + } + + // Reads one byte and returns a bool, or false when no data remains. + bool ConsumeBool() { return 1 & ConsumeIntegral<uint8_t>(); } + + // Returns a copy of a value selected from a fixed-size |array|. + template <typename T, size_t size> + T PickValueInArray(const T (&array)[size]) { + static_assert(size > 0, "The array must be non empty."); + return array[ConsumeIntegralInRange<size_t>(0, size - 1)]; + } + + template <typename T> + T PickValueInArray(std::initializer_list<const T> list) { + // static_assert(list.size() > 0, "The array must be non empty."); + return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1)); + } + + // Return an enum value. The enum must start at 0 and be contiguous. It must + // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as: + // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue }; + template <typename T> T ConsumeEnum() { + static_assert(std::is_enum<T>::value, "|T| must be an enum type."); + return static_cast<T>(ConsumeIntegralInRange<uint32_t>( + 0, static_cast<uint32_t>(T::kMaxValue))); + } + + // Reports the remaining bytes available for fuzzed input. + size_t remaining_bytes() { return remaining_bytes_; } + +private: + FuzzedDataProvider(const FuzzedDataProvider &) = delete; + FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete; + + void Advance(size_t num_bytes) { + if (num_bytes > remaining_bytes_) + abort(); + + data_ptr_ += num_bytes; + remaining_bytes_ -= num_bytes; + } + + template <typename T> + std::vector<T> ConsumeBytes(size_t size, size_t num_bytes_to_consume) { + static_assert(sizeof(T) == sizeof(uint8_t), "Incompatible data type."); + + // The point of using the size-based constructor below is to increase the + // odds of having a vector object with capacity being equal to the length. + // That part is always implementation specific, but at least both libc++ and + // libstdc++ allocate the requested number of bytes in that constructor, + // which seems to be a natural choice for other implementations as well. + // To increase the odds even more, we also call |shrink_to_fit| below. + std::vector<T> result(size); + std::memcpy(result.data(), data_ptr_, num_bytes_to_consume); + Advance(num_bytes_to_consume); + + // Even though |shrink_to_fit| is also implementation specific, we expect it + // to provide an additional assurance in case vector's constructor allocated + // a buffer which is larger than the actual amount of data we put inside it. + result.shrink_to_fit(); + return result; + } + + template <typename TS, typename TU> TS ConvertUnsignedToSigned(TU value) { + static_assert(sizeof(TS) == sizeof(TU), "Incompatible data types."); + static_assert(!std::numeric_limits<TU>::is_signed, + "Source type must be unsigned."); + + // TODO(Dor1s): change to `if constexpr` once C++17 becomes mainstream. + if (std::numeric_limits<TS>::is_modulo) + return static_cast<TS>(value); + + // Avoid using implementation-defined unsigned to signer conversions. + // To learn more, see https://stackoverflow.com/questions/13150449. + if (value <= std::numeric_limits<TS>::max()) + return static_cast<TS>(value); + else { + constexpr auto TS_min = std::numeric_limits<TS>::min(); + return TS_min + static_cast<char>(value - TS_min); + } + } + + const uint8_t *data_ptr_; + size_t remaining_bytes_; +}; + +#endif // LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ diff --git a/lib/gwp_asan/definitions.h b/lib/gwp_asan/definitions.h new file mode 100644 index 000000000000..1190adbd4f4f --- /dev/null +++ b/lib/gwp_asan/definitions.h @@ -0,0 +1,29 @@ +//===-- gwp_asan_definitions.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_DEFINITIONS_H_ +#define GWP_ASAN_DEFINITIONS_H_ + +#define TLS_INITIAL_EXEC __thread __attribute__((tls_model("initial-exec"))) + +#ifdef LIKELY +# undef LIKELY +#endif // defined(LIKELY) +#define LIKELY(X) __builtin_expect(!!(X), 1) + +#ifdef UNLIKELY +# undef UNLIKELY +#endif // defined(UNLIKELY) +#define UNLIKELY(X) __builtin_expect(!!(X), 0) + +#ifdef ALWAYS_INLINE +# undef ALWAYS_INLINE +#endif // defined(ALWAYS_INLINE) +#define ALWAYS_INLINE inline __attribute__((always_inline)) + +#endif // GWP_ASAN_DEFINITIONS_H_ diff --git a/lib/gwp_asan/guarded_pool_allocator.cpp b/lib/gwp_asan/guarded_pool_allocator.cpp new file mode 100644 index 000000000000..7e3628eba6ff --- /dev/null +++ b/lib/gwp_asan/guarded_pool_allocator.cpp @@ -0,0 +1,510 @@ +//===-- guarded_pool_allocator.cpp ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gwp_asan/guarded_pool_allocator.h" + +#include "gwp_asan/options.h" + +// RHEL creates the PRIu64 format macro (for printing uint64_t's) only when this +// macro is defined before including <inttypes.h>. +#ifndef __STDC_FORMAT_MACROS + #define __STDC_FORMAT_MACROS 1 +#endif + +#include <assert.h> +#include <inttypes.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +using AllocationMetadata = gwp_asan::GuardedPoolAllocator::AllocationMetadata; +using Error = gwp_asan::GuardedPoolAllocator::Error; + +namespace gwp_asan { +namespace { +// Forward declare the pointer to the singleton version of this class. +// Instantiated during initialisation, this allows the signal handler +// to find this class in order to deduce the root cause of failures. Must not be +// referenced by users outside this translation unit, in order to avoid +// init-order-fiasco. +GuardedPoolAllocator *SingletonPtr = nullptr; + +class ScopedBoolean { +public: + ScopedBoolean(bool &B) : Bool(B) { Bool = true; } + ~ScopedBoolean() { Bool = false; } + +private: + bool &Bool; +}; + +void defaultPrintStackTrace(uintptr_t *Trace, options::Printf_t Printf) { + if (Trace[0] == 0) + Printf(" <unknown (does your allocator support backtracing?)>\n"); + + for (size_t i = 0; Trace[i] != 0; ++i) { + Printf(" #%zu 0x%zx in <unknown>\n", i, Trace[i]); + } + Printf("\n"); +} +} // anonymous namespace + +// Gets the singleton implementation of this class. Thread-compatible until +// init() is called, thread-safe afterwards. +GuardedPoolAllocator *getSingleton() { return SingletonPtr; } + +void GuardedPoolAllocator::AllocationMetadata::RecordAllocation( + uintptr_t AllocAddr, size_t AllocSize, options::Backtrace_t Backtrace) { + Addr = AllocAddr; + Size = AllocSize; + IsDeallocated = false; + + // TODO(hctim): Ask the caller to provide the thread ID, so we don't waste + // other thread's time getting the thread ID under lock. + AllocationTrace.ThreadID = getThreadID(); + DeallocationTrace.ThreadID = kInvalidThreadID; + if (Backtrace) + Backtrace(AllocationTrace.Trace, kMaximumStackFrames); + else + AllocationTrace.Trace[0] = 0; + DeallocationTrace.Trace[0] = 0; +} + +void GuardedPoolAllocator::AllocationMetadata::RecordDeallocation( + options::Backtrace_t Backtrace) { + IsDeallocated = true; + // Ensure that the unwinder is not called if the recursive flag is set, + // otherwise non-reentrant unwinders may deadlock. + if (Backtrace && !ThreadLocals.RecursiveGuard) { + ScopedBoolean B(ThreadLocals.RecursiveGuard); + Backtrace(DeallocationTrace.Trace, kMaximumStackFrames); + } else { + DeallocationTrace.Trace[0] = 0; + } + DeallocationTrace.ThreadID = getThreadID(); +} + +void GuardedPoolAllocator::init(const options::Options &Opts) { + // Note: We return from the constructor here if GWP-ASan is not available. + // This will stop heap-allocation of class members, as well as mmap() of the + // guarded slots. + if (!Opts.Enabled || Opts.SampleRate == 0 || + Opts.MaxSimultaneousAllocations == 0) + return; + + // TODO(hctim): Add a death unit test for this. + if (SingletonPtr) { + (*SingletonPtr->Printf)( + "GWP-ASan Error: init() has already been called.\n"); + exit(EXIT_FAILURE); + } + + if (Opts.SampleRate < 0) { + Opts.Printf("GWP-ASan Error: SampleRate is < 0.\n"); + exit(EXIT_FAILURE); + } + + if (Opts.SampleRate > INT32_MAX) { + Opts.Printf("GWP-ASan Error: SampleRate is > 2^31.\n"); + exit(EXIT_FAILURE); + } + + if (Opts.MaxSimultaneousAllocations < 0) { + Opts.Printf("GWP-ASan Error: MaxSimultaneousAllocations is < 0.\n"); + exit(EXIT_FAILURE); + } + + SingletonPtr = this; + + MaxSimultaneousAllocations = Opts.MaxSimultaneousAllocations; + + PageSize = getPlatformPageSize(); + + PerfectlyRightAlign = Opts.PerfectlyRightAlign; + Printf = Opts.Printf; + Backtrace = Opts.Backtrace; + if (Opts.PrintBacktrace) + PrintBacktrace = Opts.PrintBacktrace; + else + PrintBacktrace = defaultPrintStackTrace; + + size_t PoolBytesRequired = + PageSize * (1 + MaxSimultaneousAllocations) + + MaxSimultaneousAllocations * maximumAllocationSize(); + void *GuardedPoolMemory = mapMemory(PoolBytesRequired); + + size_t BytesRequired = MaxSimultaneousAllocations * sizeof(*Metadata); + Metadata = reinterpret_cast<AllocationMetadata *>(mapMemory(BytesRequired)); + markReadWrite(Metadata, BytesRequired); + + // Allocate memory and set up the free pages queue. + BytesRequired = MaxSimultaneousAllocations * sizeof(*FreeSlots); + FreeSlots = reinterpret_cast<size_t *>(mapMemory(BytesRequired)); + markReadWrite(FreeSlots, BytesRequired); + + // Multiply the sample rate by 2 to give a good, fast approximation for (1 / + // SampleRate) chance of sampling. + if (Opts.SampleRate != 1) + AdjustedSampleRate = static_cast<uint32_t>(Opts.SampleRate) * 2; + else + AdjustedSampleRate = 1; + + GuardedPagePool = reinterpret_cast<uintptr_t>(GuardedPoolMemory); + GuardedPagePoolEnd = + reinterpret_cast<uintptr_t>(GuardedPoolMemory) + PoolBytesRequired; + + // Ensure that signal handlers are installed as late as possible, as the class + // is not thread-safe until init() is finished, and thus a SIGSEGV may cause a + // race to members if recieved during init(). + if (Opts.InstallSignalHandlers) + installSignalHandlers(); +} + +void *GuardedPoolAllocator::allocate(size_t Size) { + // GuardedPagePoolEnd == 0 when GWP-ASan is disabled. If we are disabled, fall + // back to the supporting allocator. + if (GuardedPagePoolEnd == 0) + return nullptr; + + // Protect against recursivity. + if (ThreadLocals.RecursiveGuard) + return nullptr; + ScopedBoolean SB(ThreadLocals.RecursiveGuard); + + if (Size == 0 || Size > maximumAllocationSize()) + return nullptr; + + size_t Index; + { + ScopedLock L(PoolMutex); + Index = reserveSlot(); + } + + if (Index == kInvalidSlotID) + return nullptr; + + uintptr_t Ptr = slotToAddr(Index); + Ptr += allocationSlotOffset(Size); + AllocationMetadata *Meta = addrToMetadata(Ptr); + + // If a slot is multiple pages in size, and the allocation takes up a single + // page, we can improve overflow detection by leaving the unused pages as + // unmapped. + markReadWrite(reinterpret_cast<void *>(getPageAddr(Ptr)), Size); + + Meta->RecordAllocation(Ptr, Size, Backtrace); + + return reinterpret_cast<void *>(Ptr); +} + +void GuardedPoolAllocator::deallocate(void *Ptr) { + assert(pointerIsMine(Ptr) && "Pointer is not mine!"); + uintptr_t UPtr = reinterpret_cast<uintptr_t>(Ptr); + uintptr_t SlotStart = slotToAddr(addrToSlot(UPtr)); + AllocationMetadata *Meta = addrToMetadata(UPtr); + if (Meta->Addr != UPtr) { + reportError(UPtr, Error::INVALID_FREE); + exit(EXIT_FAILURE); + } + + // Intentionally scope the mutex here, so that other threads can access the + // pool during the expensive markInaccessible() call. + { + ScopedLock L(PoolMutex); + if (Meta->IsDeallocated) { + reportError(UPtr, Error::DOUBLE_FREE); + exit(EXIT_FAILURE); + } + + // Ensure that the deallocation is recorded before marking the page as + // inaccessible. Otherwise, a racy use-after-free will have inconsistent + // metadata. + Meta->RecordDeallocation(Backtrace); + } + + markInaccessible(reinterpret_cast<void *>(SlotStart), + maximumAllocationSize()); + + // And finally, lock again to release the slot back into the pool. + ScopedLock L(PoolMutex); + freeSlot(addrToSlot(UPtr)); +} + +size_t GuardedPoolAllocator::getSize(const void *Ptr) { + assert(pointerIsMine(Ptr)); + ScopedLock L(PoolMutex); + AllocationMetadata *Meta = addrToMetadata(reinterpret_cast<uintptr_t>(Ptr)); + assert(Meta->Addr == reinterpret_cast<uintptr_t>(Ptr)); + return Meta->Size; +} + +size_t GuardedPoolAllocator::maximumAllocationSize() const { return PageSize; } + +AllocationMetadata *GuardedPoolAllocator::addrToMetadata(uintptr_t Ptr) const { + return &Metadata[addrToSlot(Ptr)]; +} + +size_t GuardedPoolAllocator::addrToSlot(uintptr_t Ptr) const { + assert(pointerIsMine(reinterpret_cast<void *>(Ptr))); + size_t ByteOffsetFromPoolStart = Ptr - GuardedPagePool; + return ByteOffsetFromPoolStart / (maximumAllocationSize() + PageSize); +} + +uintptr_t GuardedPoolAllocator::slotToAddr(size_t N) const { + return GuardedPagePool + (PageSize * (1 + N)) + (maximumAllocationSize() * N); +} + +uintptr_t GuardedPoolAllocator::getPageAddr(uintptr_t Ptr) const { + assert(pointerIsMine(reinterpret_cast<void *>(Ptr))); + return Ptr & ~(static_cast<uintptr_t>(PageSize) - 1); +} + +bool GuardedPoolAllocator::isGuardPage(uintptr_t Ptr) const { + assert(pointerIsMine(reinterpret_cast<void *>(Ptr))); + size_t PageOffsetFromPoolStart = (Ptr - GuardedPagePool) / PageSize; + size_t PagesPerSlot = maximumAllocationSize() / PageSize; + return (PageOffsetFromPoolStart % (PagesPerSlot + 1)) == 0; +} + +size_t GuardedPoolAllocator::reserveSlot() { + // Avoid potential reuse of a slot before we have made at least a single + // allocation in each slot. Helps with our use-after-free detection. + if (NumSampledAllocations < MaxSimultaneousAllocations) + return NumSampledAllocations++; + + if (FreeSlotsLength == 0) + return kInvalidSlotID; + + size_t ReservedIndex = getRandomUnsigned32() % FreeSlotsLength; + size_t SlotIndex = FreeSlots[ReservedIndex]; + FreeSlots[ReservedIndex] = FreeSlots[--FreeSlotsLength]; + return SlotIndex; +} + +void GuardedPoolAllocator::freeSlot(size_t SlotIndex) { + assert(FreeSlotsLength < MaxSimultaneousAllocations); + FreeSlots[FreeSlotsLength++] = SlotIndex; +} + +uintptr_t GuardedPoolAllocator::allocationSlotOffset(size_t Size) const { + assert(Size > 0); + + bool ShouldRightAlign = getRandomUnsigned32() % 2 == 0; + if (!ShouldRightAlign) + return 0; + + uintptr_t Offset = maximumAllocationSize(); + if (!PerfectlyRightAlign) { + if (Size == 3) + Size = 4; + else if (Size > 4 && Size <= 8) + Size = 8; + else if (Size > 8 && (Size % 16) != 0) + Size += 16 - (Size % 16); + } + Offset -= Size; + return Offset; +} + +void GuardedPoolAllocator::reportError(uintptr_t AccessPtr, Error E) { + if (SingletonPtr) + SingletonPtr->reportErrorInternal(AccessPtr, E); +} + +size_t GuardedPoolAllocator::getNearestSlot(uintptr_t Ptr) const { + if (Ptr <= GuardedPagePool + PageSize) + return 0; + if (Ptr > GuardedPagePoolEnd - PageSize) + return MaxSimultaneousAllocations - 1; + + if (!isGuardPage(Ptr)) + return addrToSlot(Ptr); + + if (Ptr % PageSize <= PageSize / 2) + return addrToSlot(Ptr - PageSize); // Round down. + return addrToSlot(Ptr + PageSize); // Round up. +} + +Error GuardedPoolAllocator::diagnoseUnknownError(uintptr_t AccessPtr, + AllocationMetadata **Meta) { + // Let's try and figure out what the source of this error is. + if (isGuardPage(AccessPtr)) { + size_t Slot = getNearestSlot(AccessPtr); + AllocationMetadata *SlotMeta = addrToMetadata(slotToAddr(Slot)); + + // Ensure that this slot was allocated once upon a time. + if (!SlotMeta->Addr) + return Error::UNKNOWN; + *Meta = SlotMeta; + + if (SlotMeta->Addr < AccessPtr) + return Error::BUFFER_OVERFLOW; + return Error::BUFFER_UNDERFLOW; + } + + // Access wasn't a guard page, check for use-after-free. + AllocationMetadata *SlotMeta = addrToMetadata(AccessPtr); + if (SlotMeta->IsDeallocated) { + *Meta = SlotMeta; + return Error::USE_AFTER_FREE; + } + + // If we have reached here, the error is still unknown. There is no metadata + // available. + *Meta = nullptr; + return Error::UNKNOWN; +} + +namespace { +// Prints the provided error and metadata information. +void printErrorType(Error E, uintptr_t AccessPtr, AllocationMetadata *Meta, + options::Printf_t Printf, uint64_t ThreadID) { + // Print using intermediate strings. Platforms like Android don't like when + // you print multiple times to the same line, as there may be a newline + // appended to a log file automatically per Printf() call. + const char *ErrorString; + switch (E) { + case Error::UNKNOWN: + ErrorString = "GWP-ASan couldn't automatically determine the source of " + "the memory error. It was likely caused by a wild memory " + "access into the GWP-ASan pool. The error occured"; + break; + case Error::USE_AFTER_FREE: + ErrorString = "Use after free"; + break; + case Error::DOUBLE_FREE: + ErrorString = "Double free"; + break; + case Error::INVALID_FREE: + ErrorString = "Invalid (wild) free"; + break; + case Error::BUFFER_OVERFLOW: + ErrorString = "Buffer overflow"; + break; + case Error::BUFFER_UNDERFLOW: + ErrorString = "Buffer underflow"; + break; + } + + constexpr size_t kDescriptionBufferLen = 128; + char DescriptionBuffer[kDescriptionBufferLen]; + if (Meta) { + if (E == Error::USE_AFTER_FREE) { + snprintf(DescriptionBuffer, kDescriptionBufferLen, + "(%zu byte%s into a %zu-byte allocation at 0x%zx)", + AccessPtr - Meta->Addr, (AccessPtr - Meta->Addr == 1) ? "" : "s", + Meta->Size, Meta->Addr); + } else if (AccessPtr < Meta->Addr) { + snprintf(DescriptionBuffer, kDescriptionBufferLen, + "(%zu byte%s to the left of a %zu-byte allocation at 0x%zx)", + Meta->Addr - AccessPtr, (Meta->Addr - AccessPtr == 1) ? "" : "s", + Meta->Size, Meta->Addr); + } else if (AccessPtr > Meta->Addr) { + snprintf(DescriptionBuffer, kDescriptionBufferLen, + "(%zu byte%s to the right of a %zu-byte allocation at 0x%zx)", + AccessPtr - Meta->Addr, (AccessPtr - Meta->Addr == 1) ? "" : "s", + Meta->Size, Meta->Addr); + } else { + snprintf(DescriptionBuffer, kDescriptionBufferLen, + "(a %zu-byte allocation)", Meta->Size); + } + } + + // Possible number of digits of a 64-bit number: ceil(log10(2^64)) == 20. Add + // a null terminator, and round to the nearest 8-byte boundary. + constexpr size_t kThreadBufferLen = 24; + char ThreadBuffer[kThreadBufferLen]; + if (ThreadID == GuardedPoolAllocator::kInvalidThreadID) + snprintf(ThreadBuffer, kThreadBufferLen, "<unknown>"); + else + snprintf(ThreadBuffer, kThreadBufferLen, "%" PRIu64, ThreadID); + + Printf("%s at 0x%zx %s by thread %s here:\n", ErrorString, AccessPtr, + DescriptionBuffer, ThreadBuffer); +} + +void printAllocDeallocTraces(uintptr_t AccessPtr, AllocationMetadata *Meta, + options::Printf_t Printf, + options::PrintBacktrace_t PrintBacktrace) { + assert(Meta != nullptr && "Metadata is non-null for printAllocDeallocTraces"); + + if (Meta->IsDeallocated) { + if (Meta->DeallocationTrace.ThreadID == + GuardedPoolAllocator::kInvalidThreadID) + Printf("0x%zx was deallocated by thread <unknown> here:\n", AccessPtr); + else + Printf("0x%zx was deallocated by thread %zu here:\n", AccessPtr, + Meta->DeallocationTrace.ThreadID); + + PrintBacktrace(Meta->DeallocationTrace.Trace, Printf); + } + + if (Meta->AllocationTrace.ThreadID == GuardedPoolAllocator::kInvalidThreadID) + Printf("0x%zx was allocated by thread <unknown> here:\n", Meta->Addr); + else + Printf("0x%zx was allocated by thread %zu here:\n", Meta->Addr, + Meta->AllocationTrace.ThreadID); + + PrintBacktrace(Meta->AllocationTrace.Trace, Printf); +} + +struct ScopedEndOfReportDecorator { + ScopedEndOfReportDecorator(options::Printf_t Printf) : Printf(Printf) {} + ~ScopedEndOfReportDecorator() { Printf("*** End GWP-ASan report ***\n"); } + options::Printf_t Printf; +}; +} // anonymous namespace + +void GuardedPoolAllocator::reportErrorInternal(uintptr_t AccessPtr, Error E) { + if (!pointerIsMine(reinterpret_cast<void *>(AccessPtr))) { + return; + } + + // Attempt to prevent races to re-use the same slot that triggered this error. + // This does not guarantee that there are no races, because another thread can + // take the locks during the time that the signal handler is being called. + PoolMutex.tryLock(); + ThreadLocals.RecursiveGuard = true; + + Printf("*** GWP-ASan detected a memory error ***\n"); + ScopedEndOfReportDecorator Decorator(Printf); + + AllocationMetadata *Meta = nullptr; + + if (E == Error::UNKNOWN) { + E = diagnoseUnknownError(AccessPtr, &Meta); + } else { + size_t Slot = getNearestSlot(AccessPtr); + Meta = addrToMetadata(slotToAddr(Slot)); + // Ensure that this slot has been previously allocated. + if (!Meta->Addr) + Meta = nullptr; + } + + // Print the error information. + uint64_t ThreadID = getThreadID(); + printErrorType(E, AccessPtr, Meta, Printf, ThreadID); + if (Backtrace) { + static constexpr unsigned kMaximumStackFramesForCrashTrace = 128; + uintptr_t Trace[kMaximumStackFramesForCrashTrace]; + Backtrace(Trace, kMaximumStackFramesForCrashTrace); + + PrintBacktrace(Trace, Printf); + } else { + Printf(" <unknown (does your allocator support backtracing?)>\n\n"); + } + + if (Meta) + printAllocDeallocTraces(AccessPtr, Meta, Printf, PrintBacktrace); +} + +TLS_INITIAL_EXEC +GuardedPoolAllocator::ThreadLocalPackedVariables + GuardedPoolAllocator::ThreadLocals; +} // namespace gwp_asan diff --git a/lib/gwp_asan/guarded_pool_allocator.h b/lib/gwp_asan/guarded_pool_allocator.h new file mode 100644 index 000000000000..28a41110faed --- /dev/null +++ b/lib/gwp_asan/guarded_pool_allocator.h @@ -0,0 +1,265 @@ +//===-- guarded_pool_allocator.h --------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_ +#define GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_ + +#include "gwp_asan/definitions.h" +#include "gwp_asan/mutex.h" +#include "gwp_asan/options.h" +#include "gwp_asan/random.h" + +#include <stddef.h> +#include <stdint.h> + +namespace gwp_asan { +// This class is the primary implementation of the allocator portion of GWP- +// ASan. It is the sole owner of the pool of sequentially allocated guarded +// slots. It should always be treated as a singleton. + +// Functions in the public interface of this class are thread-compatible until +// init() is called, at which point they become thread-safe (unless specified +// otherwise). +class GuardedPoolAllocator { +public: + static constexpr uint64_t kInvalidThreadID = UINT64_MAX; + + enum class Error { + UNKNOWN, + USE_AFTER_FREE, + DOUBLE_FREE, + INVALID_FREE, + BUFFER_OVERFLOW, + BUFFER_UNDERFLOW + }; + + struct AllocationMetadata { + // Maximum number of stack trace frames to collect for allocations + frees. + // TODO(hctim): Implement stack frame compression, a-la Chromium. + static constexpr size_t kMaximumStackFrames = 64; + + // Records the given allocation metadata into this struct. + void RecordAllocation(uintptr_t Addr, size_t Size, + options::Backtrace_t Backtrace); + + // Record that this allocation is now deallocated. + void RecordDeallocation(options::Backtrace_t Backtrace); + + struct CallSiteInfo { + // The backtrace to the allocation/deallocation. If the first value is + // zero, we did not collect a trace. + uintptr_t Trace[kMaximumStackFrames] = {}; + // The thread ID for this trace, or kInvalidThreadID if not available. + uint64_t ThreadID = kInvalidThreadID; + }; + + // The address of this allocation. + uintptr_t Addr = 0; + // Represents the actual size of the allocation. + size_t Size = 0; + + CallSiteInfo AllocationTrace; + CallSiteInfo DeallocationTrace; + + // Whether this allocation has been deallocated yet. + bool IsDeallocated = false; + }; + + // During program startup, we must ensure that memory allocations do not land + // in this allocation pool if the allocator decides to runtime-disable + // GWP-ASan. The constructor value-initialises the class such that if no + // further initialisation takes place, calls to shouldSample() and + // pointerIsMine() will return false. + constexpr GuardedPoolAllocator(){}; + GuardedPoolAllocator(const GuardedPoolAllocator &) = delete; + GuardedPoolAllocator &operator=(const GuardedPoolAllocator &) = delete; + + // Note: This class is expected to be a singleton for the lifetime of the + // program. If this object is initialised, it will leak the guarded page pool + // and metadata allocations during destruction. We can't clean up these areas + // as this may cause a use-after-free on shutdown. + ~GuardedPoolAllocator() = default; + + // Initialise the rest of the members of this class. Create the allocation + // pool using the provided options. See options.inc for runtime configuration + // options. + void init(const options::Options &Opts); + + // Return whether the allocation should be randomly chosen for sampling. + ALWAYS_INLINE bool shouldSample() { + // NextSampleCounter == 0 means we "should regenerate the counter". + // == 1 means we "should sample this allocation". + if (UNLIKELY(ThreadLocals.NextSampleCounter == 0)) + ThreadLocals.NextSampleCounter = + (getRandomUnsigned32() % AdjustedSampleRate) + 1; + + return UNLIKELY(--ThreadLocals.NextSampleCounter == 0); + } + + // Returns whether the provided pointer is a current sampled allocation that + // is owned by this pool. + ALWAYS_INLINE bool pointerIsMine(const void *Ptr) const { + uintptr_t P = reinterpret_cast<uintptr_t>(Ptr); + return GuardedPagePool <= P && P < GuardedPagePoolEnd; + } + + // Allocate memory in a guarded slot, and return a pointer to the new + // allocation. Returns nullptr if the pool is empty, the requested size is too + // large for this pool to handle, or the requested size is zero. + void *allocate(size_t Size); + + // Deallocate memory in a guarded slot. The provided pointer must have been + // allocated using this pool. This will set the guarded slot as inaccessible. + void deallocate(void *Ptr); + + // Returns the size of the allocation at Ptr. + size_t getSize(const void *Ptr); + + // Returns the largest allocation that is supported by this pool. Any + // allocations larger than this should go to the regular system allocator. + size_t maximumAllocationSize() const; + + // Dumps an error report (including allocation and deallocation stack traces). + // An optional error may be provided if the caller knows what the error is + // ahead of time. This is primarily a helper function to locate the static + // singleton pointer and call the internal version of this function. This + // method is never thread safe, and should only be called when fatal errors + // occur. + static void reportError(uintptr_t AccessPtr, Error E = Error::UNKNOWN); + + // Get the current thread ID, or kInvalidThreadID if failure. Note: This + // implementation is platform-specific. + static uint64_t getThreadID(); + +private: + static constexpr size_t kInvalidSlotID = SIZE_MAX; + + // These functions anonymously map memory or change the permissions of mapped + // memory into this process in a platform-specific way. Pointer and size + // arguments are expected to be page-aligned. These functions will never + // return on error, instead electing to kill the calling process on failure. + // Note that memory is initially mapped inaccessible. In order for RW + // mappings, call mapMemory() followed by markReadWrite() on the returned + // pointer. + void *mapMemory(size_t Size) const; + void markReadWrite(void *Ptr, size_t Size) const; + void markInaccessible(void *Ptr, size_t Size) const; + + // Get the page size from the platform-specific implementation. Only needs to + // be called once, and the result should be cached in PageSize in this class. + static size_t getPlatformPageSize(); + + // Install the SIGSEGV crash handler for printing use-after-free and heap- + // buffer-{under|over}flow exceptions. This is platform specific as even + // though POSIX and Windows both support registering handlers through + // signal(), we have to use platform-specific signal handlers to obtain the + // address that caused the SIGSEGV exception. + static void installSignalHandlers(); + + // Returns the index of the slot that this pointer resides in. If the pointer + // is not owned by this pool, the result is undefined. + size_t addrToSlot(uintptr_t Ptr) const; + + // Returns the address of the N-th guarded slot. + uintptr_t slotToAddr(size_t N) const; + + // Returns a pointer to the metadata for the owned pointer. If the pointer is + // not owned by this pool, the result is undefined. + AllocationMetadata *addrToMetadata(uintptr_t Ptr) const; + + // Returns the address of the page that this pointer resides in. + uintptr_t getPageAddr(uintptr_t Ptr) const; + + // Gets the nearest slot to the provided address. + size_t getNearestSlot(uintptr_t Ptr) const; + + // Returns whether the provided pointer is a guard page or not. The pointer + // must be within memory owned by this pool, else the result is undefined. + bool isGuardPage(uintptr_t Ptr) const; + + // Reserve a slot for a new guarded allocation. Returns kInvalidSlotID if no + // slot is available to be reserved. + size_t reserveSlot(); + + // Unreserve the guarded slot. + void freeSlot(size_t SlotIndex); + + // Returns the offset (in bytes) between the start of a guarded slot and where + // the start of the allocation should take place. Determined using the size of + // the allocation and the options provided at init-time. + uintptr_t allocationSlotOffset(size_t AllocationSize) const; + + // Returns the diagnosis for an unknown error. If the diagnosis is not + // Error::INVALID_FREE or Error::UNKNOWN, the metadata for the slot + // responsible for the error is placed in *Meta. + Error diagnoseUnknownError(uintptr_t AccessPtr, AllocationMetadata **Meta); + + void reportErrorInternal(uintptr_t AccessPtr, Error E); + + // Cached page size for this system in bytes. + size_t PageSize = 0; + + // A mutex to protect the guarded slot and metadata pool for this class. + Mutex PoolMutex; + // The number of guarded slots that this pool holds. + size_t MaxSimultaneousAllocations = 0; + // Record the number allocations that we've sampled. We store this amount so + // that we don't randomly choose to recycle a slot that previously had an + // allocation before all the slots have been utilised. + size_t NumSampledAllocations = 0; + // Pointer to the pool of guarded slots. Note that this points to the start of + // the pool (which is a guard page), not a pointer to the first guarded page. + uintptr_t GuardedPagePool = UINTPTR_MAX; + uintptr_t GuardedPagePoolEnd = 0; + // Pointer to the allocation metadata (allocation/deallocation stack traces), + // if any. + AllocationMetadata *Metadata = nullptr; + + // Pointer to an array of free slot indexes. + size_t *FreeSlots = nullptr; + // The current length of the list of free slots. + size_t FreeSlotsLength = 0; + + // See options.{h, inc} for more information. + bool PerfectlyRightAlign = false; + + // Printf function supplied by the implementing allocator. We can't (in + // general) use printf() from the cstdlib as it may malloc(), causing infinite + // recursion. + options::Printf_t Printf = nullptr; + options::Backtrace_t Backtrace = nullptr; + options::PrintBacktrace_t PrintBacktrace = nullptr; + + // The adjusted sample rate for allocation sampling. Default *must* be + // nonzero, as dynamic initialisation may call malloc (e.g. from libstdc++) + // before GPA::init() is called. This would cause an error in shouldSample(), + // where we would calculate modulo zero. This value is set UINT32_MAX, as when + // GWP-ASan is disabled, we wish to never spend wasted cycles recalculating + // the sample rate. + uint32_t AdjustedSampleRate = UINT32_MAX; + + // Pack the thread local variables into a struct to ensure that they're in + // the same cache line for performance reasons. These are the most touched + // variables in GWP-ASan. + struct alignas(8) ThreadLocalPackedVariables { + constexpr ThreadLocalPackedVariables() {} + // Thread-local decrementing counter that indicates that a given allocation + // should be sampled when it reaches zero. + uint32_t NextSampleCounter = 0; + // Guard against recursivity. Unwinders often contain complex behaviour that + // may not be safe for the allocator (i.e. the unwinder calls dlopen(), + // which calls malloc()). When recursive behaviour is detected, we will + // automatically fall back to the supporting allocator to supply the + // allocation. + bool RecursiveGuard = false; + }; + static TLS_INITIAL_EXEC ThreadLocalPackedVariables ThreadLocals; +}; +} // namespace gwp_asan + +#endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_ diff --git a/lib/gwp_asan/mutex.h b/lib/gwp_asan/mutex.h new file mode 100644 index 000000000000..c29df4cde164 --- /dev/null +++ b/lib/gwp_asan/mutex.h @@ -0,0 +1,50 @@ +//===-- mutex.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_MUTEX_H_ +#define GWP_ASAN_MUTEX_H_ + +#ifdef __unix__ +#include <pthread.h> +#else +#error "GWP-ASan is not supported on this platform." +#endif + +namespace gwp_asan { +class Mutex { +public: + constexpr Mutex() = default; + ~Mutex() = default; + Mutex(const Mutex &) = delete; + Mutex &operator=(const Mutex &) = delete; + // Lock the mutex. + void lock(); + // Nonblocking trylock of the mutex. Returns true if the lock was acquired. + bool tryLock(); + // Unlock the mutex. + void unlock(); + +private: +#ifdef __unix__ + pthread_mutex_t Mu = PTHREAD_MUTEX_INITIALIZER; +#endif // defined(__unix__) +}; + +class ScopedLock { +public: + explicit ScopedLock(Mutex &Mx) : Mu(Mx) { Mu.lock(); } + ~ScopedLock() { Mu.unlock(); } + ScopedLock(const ScopedLock &) = delete; + ScopedLock &operator=(const ScopedLock &) = delete; + +private: + Mutex Μ +}; +} // namespace gwp_asan + +#endif // GWP_ASAN_MUTEX_H_ diff --git a/lib/gwp_asan/optional/backtrace.h b/lib/gwp_asan/optional/backtrace.h new file mode 100644 index 000000000000..2700970e5e8e --- /dev/null +++ b/lib/gwp_asan/optional/backtrace.h @@ -0,0 +1,23 @@ +//===-- backtrace.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_OPTIONAL_BACKTRACE_H_ +#define GWP_ASAN_OPTIONAL_BACKTRACE_H_ + +#include "gwp_asan/options.h" + +namespace gwp_asan { +namespace options { +// Functions to get the platform-specific and implementation-specific backtrace +// and backtrace printing functions. +Backtrace_t getBacktraceFunction(); +PrintBacktrace_t getPrintBacktraceFunction(); +} // namespace options +} // namespace gwp_asan + +#endif // GWP_ASAN_OPTIONAL_BACKTRACE_H_ diff --git a/lib/gwp_asan/optional/backtrace_linux_libc.cpp b/lib/gwp_asan/optional/backtrace_linux_libc.cpp new file mode 100644 index 000000000000..f20a3100927e --- /dev/null +++ b/lib/gwp_asan/optional/backtrace_linux_libc.cpp @@ -0,0 +1,64 @@ +//===-- backtrace_linux_libc.cpp --------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <assert.h> +#include <execinfo.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "gwp_asan/optional/backtrace.h" +#include "gwp_asan/options.h" + +namespace { +void Backtrace(uintptr_t *TraceBuffer, size_t Size) { + // Grab (what seems to be) one more trace than we need. TraceBuffer needs to + // be null-terminated, but we wish to remove the frame of this function call. + static_assert(sizeof(uintptr_t) == sizeof(void *), "uintptr_t is not void*"); + int NumTraces = + backtrace(reinterpret_cast<void **>(TraceBuffer), Size); + + // Now shift the entire trace one place to the left and null-terminate. + memmove(TraceBuffer, TraceBuffer + 1, NumTraces * sizeof(void *)); + TraceBuffer[NumTraces - 1] = 0; +} + +static void PrintBacktrace(uintptr_t *Trace, + gwp_asan::options::Printf_t Printf) { + size_t NumTraces = 0; + for (; Trace[NumTraces] != 0; ++NumTraces) { + } + + if (NumTraces == 0) { + Printf(" <not found (does your allocator support backtracing?)>\n\n"); + return; + } + + char **BacktraceSymbols = + backtrace_symbols(reinterpret_cast<void **>(Trace), NumTraces); + + for (size_t i = 0; i < NumTraces; ++i) { + if (!BacktraceSymbols) + Printf(" #%zu %p\n", i, Trace[i]); + else + Printf(" #%zu %s\n", i, BacktraceSymbols[i]); + } + + Printf("\n"); + if (BacktraceSymbols) + free(BacktraceSymbols); +} +} // anonymous namespace + +namespace gwp_asan { +namespace options { +Backtrace_t getBacktraceFunction() { return Backtrace; } +PrintBacktrace_t getPrintBacktraceFunction() { return PrintBacktrace; } +} // namespace options +} // namespace gwp_asan diff --git a/lib/gwp_asan/optional/backtrace_sanitizer_common.cpp b/lib/gwp_asan/optional/backtrace_sanitizer_common.cpp new file mode 100644 index 000000000000..7d17eec0da2f --- /dev/null +++ b/lib/gwp_asan/optional/backtrace_sanitizer_common.cpp @@ -0,0 +1,69 @@ +//===-- backtrace_sanitizer_common.cpp --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include "gwp_asan/optional/backtrace.h" +#include "gwp_asan/options.h" +#include "sanitizer_common/sanitizer_stacktrace.h" + +void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp, + void *context, + bool request_fast, + u32 max_depth) { + if (!StackTrace::WillUseFastUnwind(request_fast)) { + return Unwind(max_depth, pc, bp, context, 0, 0, request_fast); + } + Unwind(max_depth, pc, 0, context, 0, 0, false); +} + +namespace { +void Backtrace(uintptr_t *TraceBuffer, size_t Size) { + __sanitizer::BufferedStackTrace Trace; + Trace.Reset(); + if (Size > __sanitizer::kStackTraceMax) + Size = __sanitizer::kStackTraceMax; + + Trace.Unwind((__sanitizer::uptr)__builtin_return_address(0), + (__sanitizer::uptr)__builtin_frame_address(0), + /* ucontext */ nullptr, + /* fast unwind */ true, Size - 1); + + memcpy(TraceBuffer, Trace.trace, Trace.size * sizeof(uintptr_t)); + TraceBuffer[Trace.size] = 0; +} + +static void PrintBacktrace(uintptr_t *Trace, + gwp_asan::options::Printf_t Printf) { + __sanitizer::StackTrace StackTrace; + StackTrace.trace = reinterpret_cast<__sanitizer::uptr *>(Trace); + + for (StackTrace.size = 0; StackTrace.size < __sanitizer::kStackTraceMax; + ++StackTrace.size) { + if (Trace[StackTrace.size] == 0) + break; + } + + if (StackTrace.size == 0) { + Printf(" <unknown (does your allocator support backtracing?)>\n\n"); + return; + } + + StackTrace.Print(); +} +} // anonymous namespace + +namespace gwp_asan { +namespace options { +Backtrace_t getBacktraceFunction() { return Backtrace; } +PrintBacktrace_t getPrintBacktraceFunction() { return PrintBacktrace; } +} // namespace options +} // namespace gwp_asan diff --git a/lib/gwp_asan/optional/options_parser.cpp b/lib/gwp_asan/optional/options_parser.cpp new file mode 100644 index 000000000000..6c2167288d6c --- /dev/null +++ b/lib/gwp_asan/optional/options_parser.cpp @@ -0,0 +1,93 @@ +//===-- options_parser.cpp --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gwp_asan/optional/options_parser.h" + +#include <stdarg.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "gwp_asan/options.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_flags.h" + +namespace gwp_asan { +namespace options { +namespace { +void registerGwpAsanFlags(__sanitizer::FlagParser *parser, Options *o) { +#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &o->Name); +#include "gwp_asan/options.inc" +#undef GWP_ASAN_OPTION +} + +const char *getCompileDefinitionGwpAsanDefaultOptions() { +#ifdef GWP_ASAN_DEFAULT_OPTIONS + return SANITIZER_STRINGIFY(GWP_ASAN_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +const char *getGwpAsanDefaultOptions() { + return (__gwp_asan_default_options) ? __gwp_asan_default_options() : ""; +} + +Options *getOptionsInternal() { + static Options GwpAsanFlags; + return &GwpAsanFlags; +} +} // anonymous namespace + +void initOptions() { + __sanitizer::SetCommonFlagsDefaults(); + + Options *o = getOptionsInternal(); + o->setDefaults(); + + __sanitizer::FlagParser Parser; + registerGwpAsanFlags(&Parser, o); + + // Override from compile definition. + Parser.ParseString(getCompileDefinitionGwpAsanDefaultOptions()); + + // Override from user-specified string. + Parser.ParseString(getGwpAsanDefaultOptions()); + + // Override from environment. + Parser.ParseString(__sanitizer::GetEnv("GWP_ASAN_OPTIONS")); + + __sanitizer::InitializeCommonFlags(); + if (__sanitizer::Verbosity()) + __sanitizer::ReportUnrecognizedFlags(); + + if (!o->Enabled) + return; + + // Sanity checks for the parameters. + if (o->MaxSimultaneousAllocations <= 0) { + __sanitizer::Printf("GWP-ASan ERROR: MaxSimultaneousAllocations must be > " + "0 when GWP-ASan is enabled.\n"); + exit(EXIT_FAILURE); + } + + if (o->SampleRate < 1) { + __sanitizer::Printf( + "GWP-ASan ERROR: SampleRate must be > 0 when GWP-ASan is enabled.\n"); + exit(EXIT_FAILURE); + } + + o->Printf = __sanitizer::Printf; +} + +Options &getOptions() { return *getOptionsInternal(); } + +} // namespace options +} // namespace gwp_asan diff --git a/lib/gwp_asan/optional/options_parser.h b/lib/gwp_asan/optional/options_parser.h new file mode 100644 index 000000000000..7a6bfaf0ce3e --- /dev/null +++ b/lib/gwp_asan/optional/options_parser.h @@ -0,0 +1,31 @@ +//===-- options_parser.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_ +#define GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_ + +#include "gwp_asan/optional/backtrace.h" +#include "gwp_asan/options.h" +#include "sanitizer_common/sanitizer_common.h" + +namespace gwp_asan { +namespace options { +// Parse the options from the GWP_ASAN_FLAGS environment variable. +void initOptions(); +// Returns the initialised options. Call initOptions() prior to calling this +// function. +Options &getOptions(); +} // namespace options +} // namespace gwp_asan + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char * +__gwp_asan_default_options(); +} + +#endif // GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_ diff --git a/lib/gwp_asan/options.h b/lib/gwp_asan/options.h new file mode 100644 index 000000000000..6423e16526f4 --- /dev/null +++ b/lib/gwp_asan/options.h @@ -0,0 +1,59 @@ +//===-- options.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_OPTIONS_H_ +#define GWP_ASAN_OPTIONS_H_ + +#include <stddef.h> +#include <stdint.h> + +namespace gwp_asan { +namespace options { +// The function pointer type for printf(). Follows the standard format from the +// sanitizers library. If the supported allocator exposes printing via a +// different function signature, please provide a wrapper which has this +// printf() signature, and pass the wrapper instead. +typedef void (*Printf_t)(const char *Format, ...); + +// The function pointer type for backtrace information. Required to be +// implemented by the supporting allocator. The callee should elide itself and +// all frames below itself from TraceBuffer, i.e. the caller's frame should be +// in TraceBuffer[0], and subsequent frames 1..n into TraceBuffer[1..n], where a +// maximum of `MaximumDepth - 1` frames are stored. TraceBuffer should be +// nullptr-terminated (i.e. if there are 5 frames; TraceBuffer[5] == nullptr). +// If the allocator cannot supply backtrace information, it should set +// TraceBuffer[0] == nullptr. +typedef void (*Backtrace_t)(uintptr_t *TraceBuffer, size_t Size); +typedef void (*PrintBacktrace_t)(uintptr_t *TraceBuffer, Printf_t Print); + +struct Options { + Printf_t Printf = nullptr; + Backtrace_t Backtrace = nullptr; + PrintBacktrace_t PrintBacktrace = nullptr; + + // Read the options from the included definitions file. +#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \ + Type Name = DefaultValue; +#include "gwp_asan/options.inc" +#undef GWP_ASAN_OPTION + + void setDefaults() { +#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \ + Name = DefaultValue; +#include "gwp_asan/options.inc" +#undef GWP_ASAN_OPTION + + Printf = nullptr; + Backtrace = nullptr; + PrintBacktrace = nullptr; + } +}; +} // namespace options +} // namespace gwp_asan + +#endif // GWP_ASAN_OPTIONS_H_ diff --git a/lib/gwp_asan/options.inc b/lib/gwp_asan/options.inc new file mode 100644 index 000000000000..9042b11895ae --- /dev/null +++ b/lib/gwp_asan/options.inc @@ -0,0 +1,41 @@ +//===-- options.inc ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_OPTION +#error "Define GWP_ASAN_OPTION prior to including this file!" +#endif + +GWP_ASAN_OPTION(bool, Enabled, true, "Is GWP-ASan enabled? Defaults to true.") + +GWP_ASAN_OPTION( + bool, PerfectlyRightAlign, false, + "When allocations are right-aligned, should we perfectly align them up to " + "the page boundary? By default (false), we round up allocation size to the " + "nearest power of two (1, 2, 4, 8, 16) up to a maximum of 16-byte " + "alignment for performance reasons. Setting this to true can find single " + "byte buffer-overflows for multibyte allocations at the cost of " + "performance, and may be incompatible with some architectures.") + +GWP_ASAN_OPTION( + int, MaxSimultaneousAllocations, 16, + "Number of usable guarded slots in the allocation pool. Defaults to 16.") + +GWP_ASAN_OPTION(int, SampleRate, 5000, + "The probability (1 / SampleRate) that an allocation is " + "selected for GWP-ASan sampling. Default is 5000. Sample rates " + "up to (2^31 - 1) are supported.") + +GWP_ASAN_OPTION( + bool, InstallSignalHandlers, true, + "Install GWP-ASan signal handlers for SIGSEGV during dynamic loading. This " + "allows better error reports by providing stack traces for allocation and " + "deallocation when reporting a memory error. GWP-ASan's signal handler " + "will forward the signal to any previously-installed handler, and user " + "programs that install further signal handlers should make sure they do " + "the same. Note, if the previously installed SIGSEGV handler is SIG_IGN, " + "we terminate the process after dumping the error report.") diff --git a/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp b/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp new file mode 100644 index 000000000000..8bc0aefeec44 --- /dev/null +++ b/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp @@ -0,0 +1,96 @@ +//===-- guarded_pool_allocator_posix.cpp ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gwp_asan/guarded_pool_allocator.h" + +#include <stdlib.h> +#include <errno.h> +#include <signal.h> +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <unistd.h> + +namespace gwp_asan { + +void *GuardedPoolAllocator::mapMemory(size_t Size) const { + void *Ptr = + mmap(nullptr, Size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + + if (Ptr == MAP_FAILED) { + Printf("Failed to map guarded pool allocator memory, errno: %d\n", errno); + Printf(" mmap(nullptr, %zu, ...) failed.\n", Size); + exit(EXIT_FAILURE); + } + return Ptr; +} + +void GuardedPoolAllocator::markReadWrite(void *Ptr, size_t Size) const { + if (mprotect(Ptr, Size, PROT_READ | PROT_WRITE) != 0) { + Printf("Failed to set guarded pool allocator memory at as RW, errno: %d\n", + errno); + Printf(" mprotect(%p, %zu, RW) failed.\n", Ptr, Size); + exit(EXIT_FAILURE); + } +} + +void GuardedPoolAllocator::markInaccessible(void *Ptr, size_t Size) const { + // mmap() a PROT_NONE page over the address to release it to the system, if + // we used mprotect() here the system would count pages in the quarantine + // against the RSS. + if (mmap(Ptr, Size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, + 0) == MAP_FAILED) { + Printf("Failed to set guarded pool allocator memory as inaccessible, " + "errno: %d\n", + errno); + Printf(" mmap(%p, %zu, NONE, ...) failed.\n", Ptr, Size); + exit(EXIT_FAILURE); + } +} + +size_t GuardedPoolAllocator::getPlatformPageSize() { + return sysconf(_SC_PAGESIZE); +} + +struct sigaction PreviousHandler; + +static void sigSegvHandler(int sig, siginfo_t *info, void *ucontext) { + gwp_asan::GuardedPoolAllocator::reportError( + reinterpret_cast<uintptr_t>(info->si_addr)); + + // Process any previous handlers. + if (PreviousHandler.sa_flags & SA_SIGINFO) { + PreviousHandler.sa_sigaction(sig, info, ucontext); + } else if (PreviousHandler.sa_handler == SIG_IGN || + PreviousHandler.sa_handler == SIG_DFL) { + // If the previous handler was the default handler, or was ignoring this + // signal, install the default handler and re-raise the signal in order to + // get a core dump and terminate this process. + signal(SIGSEGV, SIG_DFL); + raise(SIGSEGV); + } else { + PreviousHandler.sa_handler(sig); + } +} + +void GuardedPoolAllocator::installSignalHandlers() { + struct sigaction Action; + Action.sa_sigaction = sigSegvHandler; + Action.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &Action, &PreviousHandler); +} + +uint64_t GuardedPoolAllocator::getThreadID() { +#ifdef SYS_gettid + return syscall(SYS_gettid); +#else + return kInvalidThreadID; +#endif +} + +} // namespace gwp_asan diff --git a/lib/gwp_asan/platform_specific/mutex_posix.cpp b/lib/gwp_asan/platform_specific/mutex_posix.cpp new file mode 100644 index 000000000000..8bd405e1074c --- /dev/null +++ b/lib/gwp_asan/platform_specific/mutex_posix.cpp @@ -0,0 +1,30 @@ +//===-- mutex_posix.cpp -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gwp_asan/mutex.h" + +#include <assert.h> +#include <pthread.h> + +namespace gwp_asan { +void Mutex::lock() { + int Status = pthread_mutex_lock(&Mu); + assert(Status == 0); + // Remove warning for non-debug builds. + (void)Status; +} + +bool Mutex::tryLock() { return pthread_mutex_trylock(&Mu) == 0; } + +void Mutex::unlock() { + int Status = pthread_mutex_unlock(&Mu); + assert(Status == 0); + // Remove warning for non-debug builds. + (void)Status; +} +} // namespace gwp_asan diff --git a/lib/gwp_asan/random.cpp b/lib/gwp_asan/random.cpp new file mode 100644 index 000000000000..90493da7e038 --- /dev/null +++ b/lib/gwp_asan/random.cpp @@ -0,0 +1,23 @@ +//===-- random.cpp ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gwp_asan/random.h" +#include "gwp_asan/guarded_pool_allocator.h" + +#include <time.h> + +namespace gwp_asan { +uint32_t getRandomUnsigned32() { + thread_local uint32_t RandomState = + time(nullptr) + GuardedPoolAllocator::getThreadID(); + RandomState ^= RandomState << 13; + RandomState ^= RandomState >> 17; + RandomState ^= RandomState << 5; + return RandomState; +} +} // namespace gwp_asan diff --git a/lib/gwp_asan/random.h b/lib/gwp_asan/random.h new file mode 100644 index 000000000000..5fcf30d557ee --- /dev/null +++ b/lib/gwp_asan/random.h @@ -0,0 +1,20 @@ +//===-- random.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_RANDOM_H_ +#define GWP_ASAN_RANDOM_H_ + +#include <stdint.h> + +namespace gwp_asan { +// xorshift (32-bit output), extremely fast PRNG that uses arithmetic operations +// only. Seeded using walltime. +uint32_t getRandomUnsigned32(); +} // namespace gwp_asan + +#endif // GWP_ASAN_RANDOM_H_ diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cpp index e2bfea5e4226..6f2246552164 100644 --- a/lib/hwasan/hwasan.cc +++ b/lib/hwasan/hwasan.cpp @@ -1,9 +1,8 @@ -//===-- hwasan.cc ---------------------------------------------------------===// +//===-- hwasan.cpp --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "hwasan.h" #include "hwasan_checks.h" +#include "hwasan_dynamic_shadow.h" #include "hwasan_poisoning.h" #include "hwasan_report.h" #include "hwasan_thread.h" @@ -58,7 +58,7 @@ Flags *flags() { } int hwasan_inited = 0; -int hwasan_shadow_inited = 0; +int hwasan_instrumentation_inited = 0; bool hwasan_init_is_running; int hwasan_report_count = 0; @@ -88,6 +88,8 @@ static void InitializeFlags() { cf.check_printf = false; cf.intercept_tls_get_addr = true; cf.exitcode = 99; + // 8 shadow pages ~512kB, small enough to cover common stack sizes. + cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); // Sigtrap is used in error reporting. cf.handle_sigtrap = kHandleSignalExclusive; @@ -127,13 +129,10 @@ static void InitializeFlags() { ubsan_parser.ParseString(ubsan_default_options); #endif - const char *hwasan_options = GetEnv("HWASAN_OPTIONS"); - parser.ParseString(hwasan_options); + parser.ParseStringFromEnv("HWASAN_OPTIONS"); #if HWASAN_CONTAINS_UBSAN - ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); #endif - VPrintf(1, "HWASAN_OPTIONS: %s\n", - hwasan_options ? hwasan_options : "<empty>"); InitializeCommonFlags(); @@ -142,23 +141,6 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); } -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind) { - Thread *t = GetCurrentThread(); - if (!t) { - // the thread is still being created. - stack->size = 0; - return; - } - if (!StackTrace::WillUseFastUnwind(request_fast_unwind)) { - // Block reports from our interceptors during _Unwind_Backtrace. - SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); - } - stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), - request_fast_unwind); -} - static void HWAsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2) { Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, @@ -188,17 +170,13 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) { #if SANITIZER_ANDROID static char *memory_usage_buffer = nullptr; -#define PR_SET_VMA 0x53564d41 -#define PR_SET_VMA_ANON_NAME 0 - static void InitMemoryUsage() { memory_usage_buffer = (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); CHECK(memory_usage_buffer); memory_usage_buffer[0] = '\0'; - CHECK(internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, - (uptr)memory_usage_buffer, kMemoryUsageBufferSize, - (uptr)memory_usage_buffer) == 0); + DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize, + memory_usage_buffer); } void UpdateMemoryUsage() { @@ -215,56 +193,57 @@ void UpdateMemoryUsage() { void UpdateMemoryUsage() {} #endif -struct FrameDescription { - uptr PC; - const char *Descr; -}; - -struct FrameDescriptionArray { - FrameDescription *beg, *end; -}; +// Prepare to run instrumented code on the main thread. +void InitInstrumentation() { + if (hwasan_instrumentation_inited) return; -static InternalMmapVectorNoCtor<FrameDescriptionArray> AllFrames; + if (!InitShadow()) { + Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); + DumpProcessMap(); + Die(); + } -void InitFrameDescriptors(uptr b, uptr e) { - FrameDescription *beg = reinterpret_cast<FrameDescription *>(b); - FrameDescription *end = reinterpret_cast<FrameDescription *>(e); - if (beg == end) - return; - AllFrames.push_back({beg, end}); - if (Verbosity()) - for (FrameDescription *frame_descr = beg; frame_descr < end; frame_descr++) - Printf("Frame: %p %s\n", frame_descr->PC, frame_descr->Descr); -} + InitThreads(); + hwasanThreadList().CreateCurrentThread(); -const char *GetStackFrameDescr(uptr pc) { - for (uptr i = 0, n = AllFrames.size(); i < n; i++) - for (auto p = AllFrames[i].beg; p < AllFrames[i].end; p++) - if (p->PC == pc) - return p->Descr; - return nullptr; + hwasan_instrumentation_inited = 1; } } // namespace __hwasan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __hwasan; + Thread *t = GetCurrentThread(); + if (!t) { + // the thread is still being created. + size = 0; + return; + } + if (!StackTrace::WillUseFastUnwind(request_fast)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return Unwind(max_depth, pc, bp, context, 0, 0, request_fast); + } + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} + // Interface. using namespace __hwasan; uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. -void __hwasan_shadow_init() { - if (hwasan_shadow_inited) return; - if (!InitShadow()) { - Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); - DumpProcessMap(); - Die(); - } - hwasan_shadow_inited = 1; -} +// This function was used by the old frame descriptor mechanism. We keep it +// around to avoid breaking ABI. +void __hwasan_init_frames(uptr beg, uptr end) {} -void __hwasan_init_frames(uptr beg, uptr end) { - InitFrameDescriptors(beg, end); +void __hwasan_init_static() { + InitShadowGOT(); + InitInstrumentation(); } void __hwasan_init() { @@ -287,10 +266,11 @@ void __hwasan_init() { DisableCoreDumperIfNecessary(); - __hwasan_shadow_init(); + InitInstrumentation(); - InitThreads(); - hwasanThreadList().CreateCurrentThread(); + // Needs to be called here because flags()->random_tags might not have been + // initialized when InitInstrumentation() was called. + GetCurrentThread()->InitRandomState(); MadviseShadow(); @@ -335,14 +315,14 @@ sptr __hwasan_test_shadow(const void *p, uptr sz) { if (sz == 0) return -1; tag_t ptr_tag = GetTagFromPointer((uptr)p); - if (ptr_tag == 0) - return -1; uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); uptr shadow_first = MemToShadow(ptr_raw); uptr shadow_last = MemToShadow(ptr_raw + sz - 1); for (uptr s = shadow_first; s <= shadow_last; ++s) - if (*(tag_t*)s != ptr_tag) - return ShadowToMem(s) - ptr_raw; + if (*(tag_t *)s != ptr_tag) { + sptr offset = ShadowToMem(s) - ptr_raw; + return offset < 0 ? 0 : offset; + } return -1; } @@ -467,6 +447,28 @@ void __hwasan_handle_longjmp(const void *sp_dst) { TagMemory(sp, dst - sp, 0); } +void __hwasan_handle_vfork(const void *sp_dst) { + uptr sp = (uptr)sp_dst; + Thread *t = GetCurrentThread(); + CHECK(t); + uptr top = t->stack_top(); + uptr bottom = t->stack_bottom(); + if (top == 0 || bottom == 0 || sp < bottom || sp >= top) { + Report( + "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: " + "stack top: %zx; current %zx; bottom: %zx \n" + "False positive error reports may follow\n", + top, sp, bottom); + return; + } + TagMemory(bottom, sp - bottom, 0); +} + +extern "C" void *__hwasan_extra_spill_area() { + Thread *t = GetCurrentThread(); + return &t->vfork_spill(); +} + void __hwasan_print_memory_usage() { InternalScopedString s(kMemoryUsageBufferSize); HwasanFormatMemoryUsage(s); diff --git a/lib/hwasan/hwasan.h b/lib/hwasan/hwasan.h index ce9e904c5c69..465e56c3a8cc 100644 --- a/lib/hwasan/hwasan.h +++ b/lib/hwasan/hwasan.h @@ -1,9 +1,8 @@ //===-- hwasan.h ------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,6 +44,11 @@ const uptr kAddressTagMask = 0xFFUL << kAddressTagShift; // for threads and stack histories. This is an ABI constant. const unsigned kShadowBaseAlignment = 32; +const unsigned kRecordAddrBaseTagShift = 3; +const unsigned kRecordFPShift = 48; +const unsigned kRecordFPLShift = 4; +const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift); + static inline tag_t GetTagFromPointer(uptr p) { return p >> kAddressTagShift; } @@ -71,6 +75,7 @@ extern int hwasan_report_count; bool ProtectRange(uptr beg, uptr end); bool InitShadow(); void InitThreads(); +void InitInstrumentation(); void MadviseShadow(); char *GetProcSelfMaps(); void InitializeInterceptors(); @@ -81,6 +86,7 @@ void HwasanAllocatorThreadFinish(); void *hwasan_malloc(uptr size, StackTrace *stack); void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack); +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack); void *hwasan_valloc(uptr size, StackTrace *stack); void *hwasan_pvalloc(uptr size, StackTrace *stack); void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); @@ -92,9 +98,6 @@ void hwasan_free(void *ptr, StackTrace *stack); void InstallTrapHandler(); void InstallAtExitHandler(); -const char *GetStackOriginDescr(u32 id, uptr *pc); -const char *GetStackFrameDescr(uptr pc); - void EnterSymbolizer(); void ExitSymbolizer(); bool IsInSymbolizer(); @@ -104,9 +107,6 @@ struct SymbolizerScope { ~SymbolizerScope() { ExitSymbolizer(); } }; -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind); - // Returns a "chained" origin id, pointing to the given stack trace followed by // the previous origin id. u32 ChainOrigin(u32 id, StackTrace *stack); @@ -115,16 +115,15 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_MALLOC_STACK_TRACE \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ - common_flags()->fast_unwind_on_malloc) + if (hwasan_inited) \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ + nullptr, common_flags()->fast_unwind_on_malloc, \ + common_flags()->malloc_context_size) #define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) + if (hwasan_inited) \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_FATAL_STACK_TRACE_HERE \ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()) diff --git a/lib/hwasan/hwasan_allocator.cc b/lib/hwasan/hwasan_allocator.cpp index 8487ed7e1e55..b4fae5820d0a 100644 --- a/lib/hwasan/hwasan_allocator.cc +++ b/lib/hwasan/hwasan_allocator.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_allocator.cc ------------------------- ---------------------===// +//===-- hwasan_allocator.cpp ------------------------ ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,7 +16,9 @@ #include "sanitizer_common/sanitizer_stackdepot.h" #include "hwasan.h" #include "hwasan_allocator.h" +#include "hwasan_checks.h" #include "hwasan_mapping.h" +#include "hwasan_malloc_bisect.h" #include "hwasan_thread.h" #include "hwasan_report.h" @@ -42,13 +43,8 @@ enum RightAlignMode { kRightAlignAlways }; -// These two variables are initialized from flags()->malloc_align_right -// in HwasanAllocatorInit and are never changed afterwards. -static RightAlignMode right_align_mode = kRightAlignNever; -static bool right_align_8 = false; - // Initialized in HwasanAllocatorInit, an never changed. -static ALIGNED(16) u8 tail_magic[kShadowAlignment]; +static ALIGNED(16) u8 tail_magic[kShadowAlignment - 1]; bool HwasanChunkView::IsAllocated() const { return metadata_ && metadata_->alloc_context_id && metadata_->requested_size; @@ -58,8 +54,6 @@ bool HwasanChunkView::IsAllocated() const { static uptr AlignRight(uptr addr, uptr requested_size) { uptr tail_size = requested_size % kShadowAlignment; if (!tail_size) return addr; - if (right_align_8) - return tail_size > 8 ? addr : addr + 8; return addr + kShadowAlignment - tail_size; } @@ -95,30 +89,7 @@ void HwasanAllocatorInit() { !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); allocator.Init(common_flags()->allocator_release_to_os_interval_ms); - switch (flags()->malloc_align_right) { - case 0: break; - case 1: - right_align_mode = kRightAlignSometimes; - right_align_8 = false; - break; - case 2: - right_align_mode = kRightAlignAlways; - right_align_8 = false; - break; - case 8: - right_align_mode = kRightAlignSometimes; - right_align_8 = true; - break; - case 9: - right_align_mode = kRightAlignAlways; - right_align_8 = true; - break; - default: - Report("ERROR: unsupported value of malloc_align_right flag: %d\n", - flags()->malloc_align_right); - Die(); - } - for (uptr i = 0; i < kShadowAlignment; i++) + for (uptr i = 0; i < sizeof(tail_magic); i++) tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); } @@ -172,23 +143,32 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size); internal_memset(allocated, flags()->malloc_fill_byte, fill_size); } - if (!right_align_mode) + if (size != orig_size) { internal_memcpy(reinterpret_cast<u8 *>(allocated) + orig_size, tail_magic, - size - orig_size); + size - orig_size - 1); + } void *user_ptr = allocated; - if (flags()->tag_in_malloc && - atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - user_ptr = (void *)TagMemoryAligned( - (uptr)user_ptr, size, t ? t->GenerateRandomTag() : kFallbackAllocTag); - - if ((orig_size % kShadowAlignment) && (alignment <= kShadowAlignment) && - right_align_mode) { - uptr as_uptr = reinterpret_cast<uptr>(user_ptr); - if (right_align_mode == kRightAlignAlways || - GetTagFromPointer(as_uptr) & 1) { // use a tag bit as a random bit. - user_ptr = reinterpret_cast<void *>(AlignRight(as_uptr, orig_size)); - meta->right_aligned = 1; + // Tagging can only be skipped when both tag_in_malloc and tag_in_free are + // false. When tag_in_malloc = false and tag_in_free = true malloc needs to + // retag to 0. + if ((flags()->tag_in_malloc || flags()->tag_in_free) && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { + if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) { + tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag; + uptr tag_size = orig_size ? orig_size : 1; + uptr full_granule_size = RoundDownTo(tag_size, kShadowAlignment); + user_ptr = + (void *)TagMemoryAligned((uptr)user_ptr, full_granule_size, tag); + if (full_granule_size != tag_size) { + u8 *short_granule = + reinterpret_cast<u8 *>(allocated) + full_granule_size; + TagMemoryAligned((uptr)short_granule, kShadowAlignment, + tag_size % kShadowAlignment); + short_granule[kShadowAlignment - 1] = tag; + } + } else { + user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0); } } @@ -198,10 +178,10 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, static bool PointerAndMemoryTagsMatch(void *tagged_ptr) { CHECK(tagged_ptr); - tag_t ptr_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr)); + uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr); tag_t mem_tag = *reinterpret_cast<tag_t *>( MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr)))); - return ptr_tag == mem_tag; + return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1); } static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { @@ -222,14 +202,15 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { // Check tail magic. uptr tagged_size = TaggedSize(orig_size); - if (flags()->free_checks_tail_magic && !right_align_mode && orig_size) { - uptr tail_size = tagged_size - orig_size; + if (flags()->free_checks_tail_magic && orig_size && + tagged_size != orig_size) { + uptr tail_size = tagged_size - orig_size - 1; CHECK_LT(tail_size, kShadowAlignment); void *tail_beg = reinterpret_cast<void *>( reinterpret_cast<uptr>(aligned_ptr) + orig_size); if (tail_size && internal_memcmp(tail_beg, tail_magic, tail_size)) ReportTailOverwritten(stack, reinterpret_cast<uptr>(tagged_ptr), - orig_size, tail_size, tail_magic); + orig_size, tail_magic); } meta->requested_size = 0; @@ -242,7 +223,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size); internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); } - if (flags()->tag_in_free && + if (flags()->tag_in_free && malloc_bisect(stack, 0) && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size), t ? t->GenerateRandomTag() : kFallbackFreeTag); @@ -335,6 +316,16 @@ void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); } +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return hwasan_realloc(ptr, nmemb * size, stack); +} + void *hwasan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull( HwasanAllocate(stack, size, GetPageSizeCached(), false)); diff --git a/lib/hwasan/hwasan_allocator.h b/lib/hwasan/hwasan_allocator.h index 6ab722fa6bef..3a50a11f3526 100644 --- a/lib/hwasan/hwasan_allocator.h +++ b/lib/hwasan/hwasan_allocator.h @@ -1,9 +1,8 @@ //===-- hwasan_allocator.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,10 +61,8 @@ struct AP64 { static const uptr kFlags = 0; }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; void AllocatorSwallowThreadLocalCache(AllocatorCache *cache); diff --git a/lib/hwasan/hwasan_checks.h b/lib/hwasan/hwasan_checks.h index 688b5e2bed82..a8de0fef20f0 100644 --- a/lib/hwasan/hwasan_checks.h +++ b/lib/hwasan/hwasan_checks.h @@ -1,9 +1,8 @@ //===-- hwasan_checks.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define HWASAN_CHECKS_H #include "hwasan_mapping.h" +#include "sanitizer_common/sanitizer_common.h" namespace __hwasan { template <unsigned X> @@ -23,8 +23,8 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { (void)p; // 0x900 is added to do not interfere with the kernel use of lower values of // brk immediate. - // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. - asm("brk %0\n\t" ::"n"(0x900 + X)); + register uptr x0 asm("x0") = p; + asm("brk %1\n\t" ::"r"(x0), "n"(0x900 + X)); #elif defined(__x86_64__) // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes // total. The pointer is passed via rdi. @@ -42,15 +42,48 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { // __builtin_unreachable(); } +// Version with access size which is not power of 2 +template <unsigned X> +__attribute__((always_inline)) static void SigTrap(uptr p, uptr size) { +#if defined(__aarch64__) + register uptr x0 asm("x0") = p; + register uptr x1 asm("x1") = size; + asm("brk %2\n\t" ::"r"(x0), "r"(x1), "n"(0x900 + X)); +#elif defined(__x86_64__) + // Size is stored in rsi. + asm volatile( + "int3\n" + "nopl %c0(%%rax)\n" ::"n"(0x40 + X), + "D"(p), "S"(size)); +#else + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + +__attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches( + tag_t mem_tag, uptr ptr, uptr sz) { + tag_t ptr_tag = GetTagFromPointer(ptr); + if (ptr_tag == mem_tag) + return true; + if (mem_tag >= kShadowAlignment) + return false; + if ((ptr & (kShadowAlignment - 1)) + sz > mem_tag) + return false; +#ifndef __aarch64__ + ptr = UntagAddr(ptr); +#endif + return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag; +} + enum class ErrorAction { Abort, Recover }; enum class AccessType { Load, Store }; template <ErrorAction EA, AccessType AT, unsigned LogSize> __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { - tag_t ptr_tag = GetTagFromPointer(p); uptr ptr_raw = p & ~kAddressTagMask; tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw); - if (UNLIKELY(ptr_tag != mem_tag)) { + if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) { SigTrap<0x20 * (EA == ErrorAction::Recover) + 0x10 * (AT == AccessType::Store) + LogSize>(p); if (EA == ErrorAction::Abort) @@ -66,15 +99,26 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, tag_t ptr_tag = GetTagFromPointer(p); uptr ptr_raw = p & ~kAddressTagMask; tag_t *shadow_first = (tag_t *)MemToShadow(ptr_raw); - tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz - 1); - for (tag_t *t = shadow_first; t <= shadow_last; ++t) + tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz); + for (tag_t *t = shadow_first; t < shadow_last; ++t) if (UNLIKELY(ptr_tag != *t)) { SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + 0xf>(p); + 0x10 * (AT == AccessType::Store) + 0xf>(p, sz); if (EA == ErrorAction::Abort) __builtin_unreachable(); } + uptr end = p + sz; + uptr tail_sz = end & 0xf; + if (UNLIKELY(tail_sz != 0 && + !PossiblyShortTagMatches( + *shadow_last, end & ~(kShadowAlignment - 1), tail_sz))) { + SigTrap<0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + 0xf>(p, sz); + if (EA == ErrorAction::Abort) + __builtin_unreachable(); + } } + } // end namespace __hwasan #endif // HWASAN_CHECKS_H diff --git a/lib/hwasan/hwasan_dynamic_shadow.cc b/lib/hwasan/hwasan_dynamic_shadow.cpp index 87670f508283..a04751f44a31 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.cc +++ b/lib/hwasan/hwasan_dynamic_shadow.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_dynamic_shadow.cc --------------------------------*- C++ -*-===// +//===-- hwasan_dynamic_shadow.cpp -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -19,6 +18,9 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_posix.h" +#include <elf.h> +#include <link.h> + // The code in this file needs to run in an unrelocated binary. It should not // access any external symbol, including its own non-hidden globals. @@ -119,10 +121,28 @@ decltype(__hwasan_shadow)* __hwasan_premap_shadow() { INTERFACE_ATTRIBUTE __attribute__((ifunc("__hwasan_premap_shadow"))) void __hwasan_shadow(); +extern __attribute((weak, visibility("hidden"))) ElfW(Rela) __rela_iplt_start[], + __rela_iplt_end[]; + } // extern "C" namespace __hwasan { +void InitShadowGOT() { + // Call the ifunc resolver for __hwasan_shadow and fill in its GOT entry. This + // needs to be done before other ifunc resolvers (which are handled by libc) + // because a resolver might read __hwasan_shadow. + typedef ElfW(Addr) (*ifunc_resolver_t)(void); + for (ElfW(Rela) *r = __rela_iplt_start; r != __rela_iplt_end; ++r) { + ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset); + ElfW(Addr) resolver = r->r_addend; + if (resolver == reinterpret_cast<ElfW(Addr)>(&__hwasan_premap_shadow)) { + *offset = reinterpret_cast<ifunc_resolver_t>(resolver)(); + break; + } + } +} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { if (IsPremapShadowAvailable()) return FindPremappedShadowStart(shadow_size_bytes); @@ -133,10 +153,12 @@ uptr FindDynamicShadowStart(uptr shadow_size_bytes) { #else namespace __hwasan { +void InitShadowGOT() {} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { return MapDynamicShadow(shadow_size_bytes); } } // namespace __hwasan -# + #endif // SANITIZER_ANDROID diff --git a/lib/hwasan/hwasan_dynamic_shadow.h b/lib/hwasan/hwasan_dynamic_shadow.h index b5e9e1dd6a06..3c2e7c716a34 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.h +++ b/lib/hwasan/hwasan_dynamic_shadow.h @@ -1,9 +1,8 @@ //===-- hwasan_dynamic_shadow.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -21,6 +20,7 @@ namespace __hwasan { uptr FindDynamicShadowStart(uptr shadow_size_bytes); +void InitShadowGOT(); } // namespace __hwasan diff --git a/lib/hwasan/hwasan_flags.h b/lib/hwasan/hwasan_flags.h index 492d5bb98c27..0a6998f675d6 100644 --- a/lib/hwasan/hwasan_flags.h +++ b/lib/hwasan/hwasan_flags.h @@ -1,9 +1,8 @@ //===-- hwasan_flags.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_flags.inc b/lib/hwasan/hwasan_flags.inc index b450ab9503f9..2dff2b9aca6e 100644 --- a/lib/hwasan/hwasan_flags.inc +++ b/lib/hwasan/hwasan_flags.inc @@ -1,9 +1,8 @@ //===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,32 +37,6 @@ HWASAN_FLAG( "HWASan allocator flag. max_malloc_fill_size is the maximal amount of " "bytes that will be filled with malloc_fill_byte on malloc.") -// Rules for malloc alignment on aarch64: -// * If the size is 16-aligned, then malloc should return 16-aligned memory. -// * Otherwise, malloc should return 8-alignment memory. -// So, -// * If the size is 16-aligned, we don't need to do anything. -// * Otherwise we don't have to obey 16-alignment, just the 8-alignment. -// * We may want to break the 8-alignment rule to catch more buffer overflows -// but this will break valid code in some rare cases, like this: -// struct Foo { -// // accessed via atomic instructions that require 8-alignment. -// std::atomic<int64_t> atomic_stuff; -// ... -// char vla[1]; // the actual size of vla could be anything. -// } -// Which means that the safe values for malloc_align_right are 0, 8, 9, -// and the values 1 and 2 may require changes in otherwise valid code. - -HWASAN_FLAG( - int, malloc_align_right, 0, // off by default - "HWASan allocator flag. " - "0 (default): allocations are always aligned left to 16-byte boundary; " - "1: allocations are sometimes aligned right to 1-byte boundary (risky); " - "2: allocations are always aligned right to 1-byte boundary (risky); " - "8: allocations are sometimes aligned right to 8-byte boundary; " - "9: allocations are always aligned right to 8-byte boundary." - ) HWASAN_FLAG(bool, free_checks_tail_magic, 1, "If set, free() will check the magic values " "to the right of the allocated object " @@ -86,3 +59,16 @@ HWASAN_FLAG(int, stack_history_size, 1024, "The number of stack frames remembered per thread. " "Affects the quality of stack-related reports, but not the ability " "to find bugs.") + +// Malloc / free bisection. Only tag malloc and free calls when a hash of +// allocation size and stack trace is between malloc_bisect_left and +// malloc_bisect_right (both inclusive). [0, 0] range is special and disables +// bisection (i.e. everything is tagged). Once the range is narrowed down +// enough, use malloc_bisect_dump to see interesting allocations. +HWASAN_FLAG(uptr, malloc_bisect_left, 0, + "Left bound of malloc bisection, inclusive.") +HWASAN_FLAG(uptr, malloc_bisect_right, 0, + "Right bound of malloc bisection, inclusive.") +HWASAN_FLAG(bool, malloc_bisect_dump, false, + "Print all allocations within [malloc_bisect_left, " + "malloc_bisect_right] range ") diff --git a/lib/hwasan/hwasan_interceptors.cc b/lib/hwasan/hwasan_interceptors.cpp index fb0dcb8905c0..47fed0fc9abb 100644 --- a/lib/hwasan/hwasan_interceptors.cc +++ b/lib/hwasan/hwasan_interceptors.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_interceptors.cc --------------------------------------------===// +//===-- hwasan_interceptors.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,24 +44,6 @@ using __sanitizer::atomic_load; using __sanitizer::atomic_store; using __sanitizer::atomic_uintptr_t; -bool IsInInterceptorScope() { - Thread *t = GetCurrentThread(); - return t && t->InInterceptorScope(); -} - -struct InterceptorScope { - InterceptorScope() { - Thread *t = GetCurrentThread(); - if (t) - t->EnterInterceptorScope(); - } - ~InterceptorScope() { - Thread *t = GetCurrentThread(); - if (t) - t->LeaveInterceptorScope(); - } -}; - static uptr allocated_for_dlsym; static const uptr kDlsymAllocPoolSize = 1024; static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; @@ -179,6 +160,11 @@ void * __sanitizer_realloc(void *ptr, uptr size) { return hwasan_realloc(ptr, size, &stack); } +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_reallocarray(ptr, nmemb, size, &stack); +} + void * __sanitizer_malloc(uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_init_is_running)) @@ -205,6 +191,7 @@ INTERCEPTOR_ALIAS(void, free, void *ptr); INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD @@ -228,6 +215,11 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, } #endif +#if HWASAN_WITH_INTERCEPTORS +DEFINE_REAL(int, vfork) +DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork) +#endif + static void BeforeFork() { StackDepotLockAll(); } @@ -244,11 +236,6 @@ INTERCEPTOR(int, fork, void) { return pid; } - -struct HwasanInterceptorContext { - bool in_interceptor_scope; -}; - namespace __hwasan { int OnExit() { @@ -267,9 +254,12 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(fork); #if HWASAN_WITH_INTERCEPTORS +#if defined(__linux__) + INTERCEPT_FUNCTION(vfork); +#endif // __linux__ #if !defined(__aarch64__) INTERCEPT_FUNCTION(pthread_create); -#endif +#endif // __aarch64__ INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); #endif diff --git a/lib/hwasan/hwasan_interceptors_vfork.S b/lib/hwasan/hwasan_interceptors_vfork.S new file mode 100644 index 000000000000..13d0829c0865 --- /dev/null +++ b/lib/hwasan/hwasan_interceptors_vfork.S @@ -0,0 +1,10 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__linux__) && HWASAN_WITH_INTERCEPTORS +#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area +#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork +#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" +#endif + +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_interface_internal.h b/lib/hwasan/hwasan_interface_internal.h index d3b2087d03c6..1b10d76c78e9 100644 --- a/lib/hwasan/hwasan_interface_internal.h +++ b/lib/hwasan/hwasan_interface_internal.h @@ -1,9 +1,8 @@ //===-- hwasan_interface_internal.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,7 +20,7 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE -void __hwasan_shadow_init(); +void __hwasan_init_static(); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_init(); @@ -101,6 +100,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __hwasan_tag_pointer(uptr p, u8 tag); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_tag_mismatch(uptr addr, u8 ts); + +SANITIZER_INTERFACE_ATTRIBUTE u8 __hwasan_generate_tag(); // Returns the offset of the first tag mismatch or -1 if the whole range is @@ -118,6 +120,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_handle_longjmp(const void *sp_dst); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_handle_vfork(const void *sp_dst); + +SANITIZER_INTERFACE_ATTRIBUTE u16 __sanitizer_unaligned_load16(const uu16 *p); SANITIZER_INTERFACE_ATTRIBUTE @@ -193,6 +198,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_realloc(void *ptr, uptr size); SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_malloc(uptr size); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/lib/hwasan/hwasan_linux.cc b/lib/hwasan/hwasan_linux.cpp index 5b0a8b4ac98f..d932976489e9 100644 --- a/lib/hwasan/hwasan_linux.cc +++ b/lib/hwasan/hwasan_linux.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_linux.cc -----------------------------------------*- C++ -*-===// +//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -39,7 +38,17 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_procmaps.h" -#if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID +// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID. +// +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF +// Not currently tested. +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON +// Integration tests downstream exist. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF +// Tested with check-hwasan on x86_64-linux. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON +// Tested with check-hwasan on aarch64-linux-android. +#if !SANITIZER_ANDROID SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL uptr __hwasan_tls; #endif @@ -219,6 +228,8 @@ bool MemIsApp(uptr p) { } static void HwasanAtExit(void) { + if (common_flags()->print_module_map) + DumpProcessMap(); if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) ReportStats(); if (hwasan_report_count > 0) { @@ -235,7 +246,7 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 extern "C" void __hwasan_thread_enter() { - hwasanThreadList().CreateCurrentThread(); + hwasanThreadList().CreateCurrentThread()->InitRandomState(); } extern "C" void __hwasan_thread_exit() { @@ -288,7 +299,9 @@ uptr *GetCurrentThreadLongPtr() { #if SANITIZER_ANDROID void AndroidTestTlsSlot() { uptr kMagicValue = 0x010203040A0B0C0D; - *(uptr *)get_android_tls_ptr() = kMagicValue; + uptr *tls_ptr = GetCurrentThreadLongPtr(); + uptr old_value = *tls_ptr; + *tls_ptr = kMagicValue; dlerror(); if (*(uptr *)get_android_tls_ptr() != kMagicValue) { Printf( @@ -296,6 +309,7 @@ void AndroidTestTlsSlot() { "for dlerror().\n"); Die(); } + *tls_ptr = old_value; } #else void AndroidTestTlsSlot() {} @@ -369,22 +383,35 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { return AccessInfo{addr, size, is_store, !is_store, recover}; } +static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, + ucontext_t *uc, uptr *registers_frame = nullptr) { + InternalMmapVector<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); + + // The second stack frame contains the failure __hwasan_check function, as + // we have a stack frame for the registers saved in __hwasan_tag_mismatch that + // we wish to ignore. This (currently) only occurs on AArch64, as x64 + // implementations use SIGTRAP to implement the failure, and thus do not go + // through the stack saver. + if (registers_frame && stack->trace && stack->size > 0) { + stack->trace++; + stack->size--; + } + + bool fatal = flags()->halt_on_error || !ai.recover; + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, + registers_frame); +} + static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { AccessInfo ai = GetAccessInfo(info, uc); if (!ai.is_store && !ai.is_load) return false; - InternalMmapVector<BufferedStackTrace> stack_buffer(1); - BufferedStackTrace *stack = stack_buffer.data(); - stack->Reset(); SignalContext sig{info, uc}; - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, uc, common_flags()->fast_unwind_on_fatal); - - ++hwasan_report_count; - - bool fatal = flags()->halt_on_error || !ai.recover; - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal); + HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc); #if defined(__aarch64__) uc->uc_mcontext.pc += 4; @@ -395,10 +422,25 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { return true; } +// Entry point stub for interoperability between __hwasan_tag_mismatch (ASM) and +// the rest of the mismatch handling code (C++). +extern "C" void __hwasan_tag_mismatch_stub(uptr addr, uptr access_info, + uptr *registers_frame) { + AccessInfo ai; + ai.is_store = access_info & 0x10; + ai.recover = false; + ai.addr = addr; + ai.size = 1 << (access_info & 0xf); + + HandleTagMismatch(ai, (uptr)__builtin_return_address(0), + (uptr)__builtin_frame_address(0), nullptr, registers_frame); + __builtin_unreachable(); +} + static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); + stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } void HwasanOnDeadlySignal(int signo, void *info, void *context) { diff --git a/lib/hwasan/hwasan_malloc_bisect.h b/lib/hwasan/hwasan_malloc_bisect.h new file mode 100644 index 000000000000..eaf124aab7dd --- /dev/null +++ b/lib/hwasan/hwasan_malloc_bisect.h @@ -0,0 +1,50 @@ +//===-- hwasan_malloc_bisect.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_hash.h" +#include "hwasan.h" + +namespace __hwasan { + +static u32 malloc_hash(StackTrace *stack, uptr orig_size) { + uptr len = Min(stack->size, (unsigned)7); + MurMur2HashBuilder H(len); + H.add(orig_size); + // Start with frame #1 to skip __sanitizer_malloc frame, which is + // (a) almost always the same (well, could be operator new or new[]) + // (b) can change hashes when compiler-rt is rebuilt, invalidating previous + // bisection results. + // Because of ASLR, use only offset inside the page. + for (uptr i = 1; i < len; ++i) H.add(((u32)stack->trace[i]) & 0xFFF); + return H.get(); +} + +static INLINE bool malloc_bisect(StackTrace *stack, uptr orig_size) { + uptr left = flags()->malloc_bisect_left; + uptr right = flags()->malloc_bisect_right; + if (LIKELY(left == 0 && right == 0)) + return true; + if (!stack) + return true; + // Allow malloc_bisect_right > (u32)(-1) to avoid spelling the latter in + // decimal. + uptr h = (uptr)malloc_hash(stack, orig_size); + if (h < left || h > right) + return false; + if (flags()->malloc_bisect_dump) { + Printf("[alloc] %u %zu\n", h, orig_size); + stack->Print(); + } + return true; +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_mapping.h b/lib/hwasan/hwasan_mapping.h index e5e23dc60337..a86ad7ca8036 100644 --- a/lib/hwasan/hwasan_mapping.h +++ b/lib/hwasan/hwasan_mapping.h @@ -1,9 +1,8 @@ //===-- hwasan_mapping.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_memintrinsics.cc b/lib/hwasan/hwasan_memintrinsics.cpp index 9cb844e45c40..e82d77a1bc16 100644 --- a/lib/hwasan/hwasan_memintrinsics.cc +++ b/lib/hwasan/hwasan_memintrinsics.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_memintrinsics.cc ---------------------------------*- C++ -*-===// +//===-- hwasan_memintrinsics.cpp --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_new_delete.cc b/lib/hwasan/hwasan_new_delete.cpp index f2e8faf5da73..4a9c79fe41b3 100644 --- a/lib/hwasan/hwasan_new_delete.cc +++ b/lib/hwasan/hwasan_new_delete.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_new_delete.cc ----------------------------------------------===// +//===-- hwasan_new_delete.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,15 +35,15 @@ namespace std { if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\ return res -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } @@ -53,13 +52,13 @@ void *operator new[](size_t size, std::nothrow_t const&) { GET_MALLOC_STACK_TRACE; \ if (ptr) hwasan_free(ptr, &stack) -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc deleted file mode 100644 index 9c8e16b12ad5..000000000000 --- a/lib/hwasan/hwasan_poisoning.cc +++ /dev/null @@ -1,37 +0,0 @@ -//===-- hwasan_poisoning.cc -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of HWAddressSanitizer. -// -//===----------------------------------------------------------------------===// - -#include "hwasan_poisoning.h" - -#include "hwasan_mapping.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" - -namespace __hwasan { - -uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { - CHECK(IsAligned(p, kShadowAlignment)); - CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MemToShadow(p); - uptr shadow_size = MemToShadowSize(size); - internal_memset((void *)shadow_start, tag, shadow_size); - return AddTagToPointer(p, tag); -} - -uptr TagMemory(uptr p, uptr size, tag_t tag) { - uptr start = RoundDownTo(p, kShadowAlignment); - uptr end = RoundUpTo(p + size, kShadowAlignment); - return TagMemoryAligned(start, end - start, tag); -} - -} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.cpp b/lib/hwasan/hwasan_poisoning.cpp new file mode 100644 index 000000000000..2a0816428e75 --- /dev/null +++ b/lib/hwasan/hwasan_poisoning.cpp @@ -0,0 +1,52 @@ +//===-- hwasan_poisoning.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "hwasan_poisoning.h" + +#include "hwasan_mapping.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_linux.h" + +namespace __hwasan { + +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); + + uptr page_size = GetPageSizeCached(); + uptr page_start = RoundUpTo(shadow_start, page_size); + uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); + uptr threshold = common_flags()->clear_shadow_mmap_threshold; + if (SANITIZER_LINUX && + UNLIKELY(page_end >= page_start + threshold && tag == 0)) { + internal_memset((void *)shadow_start, tag, page_start - shadow_start); + internal_memset((void *)page_end, tag, + shadow_start + shadow_size - page_end); + // For an anonymous private mapping MADV_DONTNEED will return a zero page on + // Linux. + ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); + } else { + internal_memset((void *)shadow_start, tag, shadow_size); + } + return AddTagToPointer(p, tag); +} + +uptr TagMemory(uptr p, uptr size, tag_t tag) { + uptr start = RoundDownTo(p, kShadowAlignment); + uptr end = RoundUpTo(p + size, kShadowAlignment); + return TagMemoryAligned(start, end - start, tag); +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.h b/lib/hwasan/hwasan_poisoning.h index 0dbf9d8ed4ef..61751f7d252e 100644 --- a/lib/hwasan/hwasan_poisoning.h +++ b/lib/hwasan/hwasan_poisoning.h @@ -1,9 +1,8 @@ //===-- hwasan_poisoning.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_report.cc b/lib/hwasan/hwasan_report.cpp index ea3e4096daef..346889797888 100644 --- a/lib/hwasan/hwasan_report.cc +++ b/lib/hwasan/hwasan_report.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_report.cc --------------------------------------------------===// +//===-- hwasan_report.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_report.h" #include "hwasan_thread.h" #include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" @@ -35,15 +35,21 @@ class ScopedReport { ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) { BlockingMutexLock lock(&error_message_lock_); error_message_ptr_ = fatal ? &error_message_ : nullptr; + ++hwasan_report_count; } ~ScopedReport() { - BlockingMutexLock lock(&error_message_lock_); - if (fatal) { - SetAbortMessage(error_message_.data()); - Die(); + { + BlockingMutexLock lock(&error_message_lock_); + if (fatal) + SetAbortMessage(error_message_.data()); + error_message_ptr_ = nullptr; } - error_message_ptr_ = nullptr; + if (common_flags()->print_module_map >= 2 || + (fatal && common_flags()->print_module_map)) + DumpProcessMap(); + if (fatal) + Die(); } static void MaybeAppendToErrorMessage(const char *msg) { @@ -133,6 +139,88 @@ uptr FindHeapAllocation(HeapAllocationsRingBuffer *rb, return 0; } +static void PrintStackAllocations(StackAllocationsRingBuffer *sa, + tag_t addr_tag, uptr untagged_addr) { + uptr frames = Min((uptr)flags()->stack_history_size, sa->size()); + bool found_local = false; + for (uptr i = 0; i < frames; i++) { + const uptr *record_addr = &(*sa)[i]; + uptr record = *record_addr; + if (!record) + break; + tag_t base_tag = + reinterpret_cast<uptr>(record_addr) >> kRecordAddrBaseTagShift; + uptr fp = (record >> kRecordFPShift) << kRecordFPLShift; + uptr pc_mask = (1ULL << kRecordFPShift) - 1; + uptr pc = record & pc_mask; + FrameInfo frame; + if (Symbolizer::GetOrInit()->SymbolizeFrame(pc, &frame)) { + for (LocalInfo &local : frame.locals) { + if (!local.has_frame_offset || !local.has_size || !local.has_tag_offset) + continue; + tag_t obj_tag = base_tag ^ local.tag_offset; + if (obj_tag != addr_tag) + continue; + // Calculate the offset from the object address to the faulting + // address. Because we only store bits 4-19 of FP (bits 0-3 are + // guaranteed to be zero), the calculation is performed mod 2^20 and may + // harmlessly underflow if the address mod 2^20 is below the object + // address. + uptr obj_offset = + (untagged_addr - fp - local.frame_offset) & (kRecordFPModulus - 1); + if (obj_offset >= local.size) + continue; + if (!found_local) { + Printf("Potentially referenced stack objects:\n"); + found_local = true; + } + Printf(" %s in %s %s:%d\n", local.name, local.function_name, + local.decl_file, local.decl_line); + } + frame.Clear(); + } + } + + if (found_local) + return; + + // We didn't find any locals. Most likely we don't have symbols, so dump + // the information that we have for offline analysis. + InternalScopedString frame_desc(GetPageSizeCached() * 2); + Printf("Previously allocated frames:\n"); + for (uptr i = 0; i < frames; i++) { + const uptr *record_addr = &(*sa)[i]; + uptr record = *record_addr; + if (!record) + break; + uptr pc_mask = (1ULL << 48) - 1; + uptr pc = record & pc_mask; + frame_desc.append(" record_addr:0x%zx record:0x%zx", + reinterpret_cast<uptr>(record_addr), record); + if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { + RenderFrame(&frame_desc, " %F %L\n", 0, frame->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + frame->ClearAll(); + } + Printf("%s", frame_desc.data()); + frame_desc.clear(); + } +} + +// Returns true if tag == *tag_ptr, reading tags from short granules if +// necessary. This may return a false positive if tags 1-15 are used as a +// regular tag rather than a short granule marker. +static bool TagsEqual(tag_t tag, tag_t *tag_ptr) { + if (tag == *tag_ptr) + return true; + if (*tag_ptr == 0 || *tag_ptr > kShadowAlignment - 1) + return false; + uptr mem = ShadowToMem(reinterpret_cast<uptr>(tag_ptr)); + tag_t inline_tag = *reinterpret_cast<tag_t *>(mem + kShadowAlignment - 1); + return tag == inline_tag; +} + void PrintAddressDescription( uptr tagged_addr, uptr access_size, StackAllocationsRingBuffer *current_stack_allocations) { @@ -160,39 +248,36 @@ void PrintAddressDescription( // check the allocator if it has a live chunk there. tag_t addr_tag = GetTagFromPointer(tagged_addr); tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); - if (*tag_ptr != addr_tag) { // should be true usually. - tag_t *left = tag_ptr, *right = tag_ptr; - // scan left. - for (int i = 0; i < 1000 && *left == *tag_ptr; i++, left--){} - // scan right. - for (int i = 0; i < 1000 && *right == *tag_ptr; i++, right++){} - // Chose the object that has addr_tag and that is closer to addr. - tag_t *candidate = nullptr; - if (*right == addr_tag && *left == addr_tag) - candidate = right - tag_ptr < tag_ptr - left ? right : left; - else if (*right == addr_tag) - candidate = right; - else if (*left == addr_tag) + tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr; + for (int i = 0; i < 1000; i++) { + if (TagsEqual(addr_tag, left)) { candidate = left; + break; + } + --left; + if (TagsEqual(addr_tag, right)) { + candidate = right; + break; + } + ++right; + } - if (candidate) { - uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); - HwasanChunkView chunk = FindHeapChunkByAddress(mem); - if (chunk.IsAllocated()) { - Printf("%s", d.Location()); - Printf( - "%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", - untagged_addr, - candidate == left ? untagged_addr - chunk.End() - : chunk.Beg() - untagged_addr, - candidate == right ? "left" : "right", chunk.UsedSize(), - chunk.Beg(), chunk.End()); - Printf("%s", d.Allocation()); - Printf("allocated here:\n"); - Printf("%s", d.Default()); - GetStackTraceFromId(chunk.GetAllocStackId()).Print(); - num_descriptions_printed++; - } + if (candidate) { + uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); + HwasanChunkView chunk = FindHeapChunkByAddress(mem); + if (chunk.IsAllocated()) { + Printf("%s", d.Location()); + Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", + untagged_addr, + candidate == left ? untagged_addr - chunk.End() + : chunk.Beg() - untagged_addr, + candidate == left ? "right" : "left", chunk.UsedSize(), + chunk.Beg(), chunk.End()); + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + num_descriptions_printed++; } } @@ -232,33 +317,10 @@ void PrintAddressDescription( Printf("%s", d.Default()); t->Announce(); - // Temporary report section, needs to be improved. - Printf("Previously allocated frames:\n"); auto *sa = (t == GetCurrentThread() && current_stack_allocations) ? current_stack_allocations : t->stack_allocations(); - uptr frames = Min((uptr)flags()->stack_history_size, sa->size()); - InternalScopedString frame_desc(GetPageSizeCached() * 2); - for (uptr i = 0; i < frames; i++) { - uptr record = (*sa)[i]; - if (!record) - break; - uptr sp = (record >> 48) << 4; - uptr pc_mask = (1ULL << 48) - 1; - uptr pc = record & pc_mask; - if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { - frame_desc.append(" sp: 0x%zx pc: %p ", sp, pc); - RenderFrame(&frame_desc, "in %f %s:%l\n", 0, frame->info, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); - frame->ClearAll(); - if (auto Descr = GetStackFrameDescr(pc)) - frame_desc.append(" %s\n", Descr); - } - Printf("%s", frame_desc.data()); - frame_desc.clear(); - } - + PrintStackAllocations(sa, addr_tag, untagged_addr); num_descriptions_printed++; } }); @@ -273,23 +335,20 @@ void PrintAddressDescription( void ReportStats() {} -static void PrintTagsAroundAddr(tag_t *tag_ptr) { - Printf( - "Memory tags around the buggy address (one tag corresponds to %zd " - "bytes):\n", kShadowAlignment); - +static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows, + void (*print_tag)(InternalScopedString &s, + tag_t *tag)) { const uptr row_len = 16; // better be power of two. - const uptr num_rows = 17; tag_t *center_row_beg = reinterpret_cast<tag_t *>( RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len)); tag_t *beg_row = center_row_beg - row_len * (num_rows / 2); - tag_t *end_row = center_row_beg + row_len * (num_rows / 2); + tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2); InternalScopedString s(GetPageSizeCached() * 8); for (tag_t *row = beg_row; row < end_row; row += row_len) { s.append("%s", row == center_row_beg ? "=>" : " "); for (uptr i = 0; i < row_len; i++) { s.append("%s", row + i == tag_ptr ? "[" : " "); - s.append("%02x", row[i]); + print_tag(s, &row[i]); s.append("%s", row + i == tag_ptr ? "]" : " "); } s.append("%s\n", row == center_row_beg ? "<=" : " "); @@ -297,6 +356,34 @@ static void PrintTagsAroundAddr(tag_t *tag_ptr) { Printf("%s", s.data()); } +static void PrintTagsAroundAddr(tag_t *tag_ptr) { + Printf( + "Memory tags around the buggy address (one tag corresponds to %zd " + "bytes):\n", kShadowAlignment); + PrintTagInfoAroundAddr(tag_ptr, 17, [](InternalScopedString &s, tag_t *tag) { + s.append("%02x", *tag); + }); + + Printf( + "Tags for short granules around the buggy address (one tag corresponds " + "to %zd bytes):\n", + kShadowAlignment); + PrintTagInfoAroundAddr(tag_ptr, 3, [](InternalScopedString &s, tag_t *tag) { + if (*tag >= 1 && *tag <= kShadowAlignment) { + uptr granule_addr = ShadowToMem(reinterpret_cast<uptr>(tag)); + s.append("%02x", + *reinterpret_cast<u8 *>(granule_addr + kShadowAlignment - 1)); + } else { + s.append(".."); + } + }); + Printf( + "See " + "https://clang.llvm.org/docs/" + "HardwareAssistedAddressSanitizerDesign.html#short-granules for a " + "description of short granule tags\n"); +} + void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { ScopedReport R(flags()->halt_on_error); @@ -324,7 +411,8 @@ void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { } void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, - uptr tail_size, const u8 *expected) { + const u8 *expected) { + uptr tail_size = kShadowAlignment - (orig_size % kShadowAlignment); ScopedReport R(flags()->halt_on_error); Decorator d; uptr untagged_addr = UntagAddr(tagged_addr); @@ -368,11 +456,9 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, "to the right of a heap object, but within the %zd-byte granule, e.g.\n" " char *x = new char[20];\n" " x[25] = 42;\n" - "By default %s does not detect such bugs at the time of write,\n" - "but can detect them at the time of free/delete.\n" - "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0;\n" - "To enable checking at the time of access, set " - "HWASAN_OPTIONS=malloc_align_right to non-zero\n\n", + "%s does not detect such bugs in uninstrumented code at the time of write," + "\nbut can detect them at the time of free/delete.\n" + "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0\n", kShadowAlignment, SanitizerToolName); Printf("%s", s.data()); GetCurrentThread()->Announce(); @@ -384,7 +470,7 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, } void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, - bool is_store, bool fatal) { + bool is_store, bool fatal, uptr *registers_frame) { ScopedReport R(fatal); SavedStackAllocations current_stack_allocations( GetCurrentThread()->stack_allocations()); @@ -400,13 +486,21 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, Thread *t = GetCurrentThread(); + sptr offset = + __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size); + CHECK(offset >= 0 && offset < static_cast<sptr>(access_size)); tag_t ptr_tag = GetTagFromPointer(tagged_addr); - tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + tag_t *tag_ptr = + reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset)); tag_t mem_tag = *tag_ptr; + Printf("%s", d.Access()); Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n", is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, mem_tag, t->unique_id()); + if (offset != 0) + Printf("Invalid access starting at offset [%zu, %zu)\n", offset, + Min(access_size, static_cast<uptr>(offset) + (1 << kShadowScale))); Printf("%s", d.Default()); stack->Print(); @@ -417,7 +511,37 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, PrintTagsAroundAddr(tag_ptr); + if (registers_frame) + ReportRegisters(registers_frame, pc); + ReportErrorSummary(bug_type, stack); } +// See the frame breakdown defined in __hwasan_tag_mismatch (from +// hwasan_tag_mismatch_aarch64.S). +void ReportRegisters(uptr *frame, uptr pc) { + Printf("Registers where the failure occurred (pc %p):\n", pc); + + // We explicitly print a single line (4 registers/line) each iteration to + // reduce the amount of logcat error messages printed. Each Printf() will + // result in a new logcat line, irrespective of whether a newline is present, + // and so we wish to reduce the number of Printf() calls we have to make. + Printf(" x0 %016llx x1 %016llx x2 %016llx x3 %016llx\n", + frame[0], frame[1], frame[2], frame[3]); + Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n", + frame[4], frame[5], frame[6], frame[7]); + Printf(" x8 %016llx x9 %016llx x10 %016llx x11 %016llx\n", + frame[8], frame[9], frame[10], frame[11]); + Printf(" x12 %016llx x13 %016llx x14 %016llx x15 %016llx\n", + frame[12], frame[13], frame[14], frame[15]); + Printf(" x16 %016llx x17 %016llx x18 %016llx x19 %016llx\n", + frame[16], frame[17], frame[18], frame[19]); + Printf(" x20 %016llx x21 %016llx x22 %016llx x23 %016llx\n", + frame[20], frame[21], frame[22], frame[23]); + Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n", + frame[24], frame[25], frame[26], frame[27]); + Printf(" x28 %016llx x29 %016llx x30 %016llx\n", + frame[28], frame[29], frame[30]); +} + } // namespace __hwasan diff --git a/lib/hwasan/hwasan_report.h b/lib/hwasan/hwasan_report.h index 10fb20cc5be1..de86c38fc01f 100644 --- a/lib/hwasan/hwasan_report.h +++ b/lib/hwasan/hwasan_report.h @@ -1,9 +1,8 @@ //===-- hwasan_report.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -23,11 +22,11 @@ namespace __hwasan { void ReportStats(); void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store, bool fatal); + bool is_store, bool fatal, uptr *registers_frame); void ReportInvalidFree(StackTrace *stack, uptr addr); void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, - uptr tail_size, const u8 *expected); - + const u8 *expected); +void ReportRegisters(uptr *registers_frame, uptr pc); void ReportAtExitStatistics(); diff --git a/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/lib/hwasan/hwasan_tag_mismatch_aarch64.S new file mode 100644 index 000000000000..92f627480486 --- /dev/null +++ b/lib/hwasan/hwasan_tag_mismatch_aarch64.S @@ -0,0 +1,106 @@ +#include "sanitizer_common/sanitizer_asm.h" + +// The content of this file is AArch64-only: +#if defined(__aarch64__) + +// The responsibility of the HWASan entry point in compiler-rt is to primarily +// readjust the stack from the callee and save the current register values to +// the stack. +// This entry point function should be called from a __hwasan_check_* symbol. +// These are generated during a lowering pass in the backend, and are found in +// AArch64AsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for +// further information. +// The __hwasan_check_* caller of this function should have expanded the stack +// and saved the previous values of x0, x1, x29, and x30. This function will +// "consume" these saved values and treats it as part of its own stack frame. +// In this sense, the __hwasan_check_* callee and this function "share" a stack +// frame. This allows us to omit having unwinding information (.cfi_*) present +// in every __hwasan_check_* function, therefore reducing binary size. This is +// particularly important as hwasan_check_* instances are duplicated in every +// translation unit where HWASan is enabled. +// This function calls HwasanTagMismatch to step back into the C++ code that +// completes the stack unwinding and error printing. This function is is not +// permitted to return. + + +// Frame from __hwasan_check_: +// | ... | +// | ... | +// | Previous stack frames... | +// +=================================+ +// | Unused 8-bytes for maintaining | +// | 16-byte SP alignment. | +// +---------------------------------+ +// | Return address (x30) for caller | +// | of __hwasan_check_*. | +// +---------------------------------+ +// | Frame address (x29) for caller | +// | of __hwasan_check_* | +// +---------------------------------+ <-- [SP + 232] +// | ... | +// | | +// | Stack frame space for x2 - x28. | +// | | +// | ... | +// +---------------------------------+ <-- [SP + 16] +// | | +// | Saved x1, as __hwasan_check_* | +// | clobbers it. | +// +---------------------------------+ +// | Saved x0, likewise above. | +// +---------------------------------+ <-- [x30 / SP] + +// This function takes two arguments: +// * x0: The address of read/write instruction that caused HWASan check fail. +// * x1: The tag size. + +.section .text +.file "hwasan_tag_mismatch_aarch64.S" +.global __hwasan_tag_mismatch +.type __hwasan_tag_mismatch, %function +__hwasan_tag_mismatch: + CFI_STARTPROC + + // Set the CFA to be the return address for caller of __hwasan_check_*. Note + // that we do not emit CFI predicates to describe the contents of this stack + // frame, as this proxy entry point should never be debugged. The contents + // are static and are handled by the unwinder after calling + // __hwasan_tag_mismatch. The frame pointer is already correctly setup + // by __hwasan_check_*. + add x29, sp, #232 + CFI_DEF_CFA(w29, 24) + CFI_OFFSET(w30, -16) + CFI_OFFSET(w29, -24) + + // Save the rest of the registers into the preallocated space left by + // __hwasan_check. + str x28, [sp, #224] + stp x26, x27, [sp, #208] + stp x24, x25, [sp, #192] + stp x22, x23, [sp, #176] + stp x20, x21, [sp, #160] + stp x18, x19, [sp, #144] + stp x16, x17, [sp, #128] + stp x14, x15, [sp, #112] + stp x12, x13, [sp, #96] + stp x10, x11, [sp, #80] + stp x8, x9, [sp, #64] + stp x6, x7, [sp, #48] + stp x4, x5, [sp, #32] + stp x2, x3, [sp, #16] + + // Pass the address of the frame to __hwasan_tag_mismatch_stub, so that it can + // extract the saved registers from this frame without having to worry about + // finding this frame. + mov x2, sp + + bl __hwasan_tag_mismatch_stub + CFI_ENDPROC + +.Lfunc_end0: + .size __hwasan_tag_mismatch, .Lfunc_end0-__hwasan_tag_mismatch + +#endif // defined(__aarch64__) + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_thread.cc b/lib/hwasan/hwasan_thread.cpp index 631c2813eb06..cabf614c005c 100644 --- a/lib/hwasan/hwasan_thread.cc +++ b/lib/hwasan/hwasan_thread.cpp @@ -25,10 +25,18 @@ static u32 RandomSeed() { return seed; } +void Thread::InitRandomState() { + random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; + + // Push a random number of zeros onto the ring buffer so that the first stack + // tag base will be random. + for (tag_t i = 0, e = GenerateRandomTag(); i != e; ++i) + stack_allocations_->push(0); +} + void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { static u64 unique_id; unique_id_ = unique_id++; - random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; if (auto sz = flags()->heap_history_size) heap_allocations_ = HeapAllocationsRingBuffer::New(sz); diff --git a/lib/hwasan/hwasan_thread.h b/lib/hwasan/hwasan_thread.h index 4830473f4aab..42c1e9e124b0 100644 --- a/lib/hwasan/hwasan_thread.h +++ b/lib/hwasan/hwasan_thread.h @@ -1,9 +1,8 @@ //===-- hwasan_thread.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer; class Thread { public: void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + void InitRandomState(); void Destroy(); uptr stack_top() { return stack_top_; } @@ -46,10 +46,6 @@ class Thread { void EnterSymbolizer() { in_symbolizer_++; } void LeaveSymbolizer() { in_symbolizer_--; } - bool InInterceptorScope() { return in_interceptor_scope_; } - void EnterInterceptorScope() { in_interceptor_scope_++; } - void LeaveInterceptorScope() { in_interceptor_scope_--; } - AllocatorCache *allocator_cache() { return &allocator_cache_; } HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; } StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; } @@ -67,11 +63,14 @@ class Thread { Print("Thread: "); } + uptr &vfork_spill() { return vfork_spill_; } + private: // NOTE: There is no Thread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. void ClearShadowForThreadStackAndTLS(); void Print(const char *prefix); + uptr vfork_spill_; uptr stack_top_; uptr stack_bottom_; uptr tls_begin_; @@ -79,7 +78,6 @@ class Thread { unsigned in_signal_handler_; unsigned in_symbolizer_; - unsigned in_interceptor_scope_; u32 random_state_; u32 random_buffer_; diff --git a/lib/hwasan/hwasan_thread_list.cc b/lib/hwasan/hwasan_thread_list.cpp index a31eee84ed93..a31eee84ed93 100644 --- a/lib/hwasan/hwasan_thread_list.cc +++ b/lib/hwasan/hwasan_thread_list.cpp diff --git a/lib/hwasan/hwasan_thread_list.h b/lib/hwasan/hwasan_thread_list.h index 53747b51fd65..914b632d9776 100644 --- a/lib/hwasan/hwasan_thread_list.h +++ b/lib/hwasan/hwasan_thread_list.h @@ -1,9 +1,8 @@ //===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -109,38 +108,52 @@ struct ThreadStats { class HwasanThreadList { public: HwasanThreadList(uptr storage, uptr size) - : free_space_(storage), - free_space_end_(storage + size), - ring_buffer_size_(RingBufferSize()) {} + : free_space_(storage), free_space_end_(storage + size) { + // [storage, storage + size) is used as a vector of + // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements. + // Each element contains + // * a ring buffer at offset 0, + // * a Thread object at offset ring_buffer_size_. + ring_buffer_size_ = RingBufferSize(); + thread_alloc_size_ = + RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); + } Thread *CreateCurrentThread() { Thread *t; { SpinMutexLock l(&list_mutex_); t = free_list_.Pop(); - if (t) - internal_memset((void *)t, 0, sizeof(Thread) + ring_buffer_size_); - else + if (t) { + uptr start = (uptr)t - ring_buffer_size_; + internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread)); + } else { t = AllocThread(); + } live_list_.Push(t); } - t->Init((uptr)(t + 1), ring_buffer_size_); + t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_); AddThreadStats(t); return t; } + void DontNeedThread(Thread *t) { + uptr start = (uptr)t - ring_buffer_size_; + ReleaseMemoryPagesToOS(start, start + thread_alloc_size_); + } + void ReleaseThread(Thread *t) { - // FIXME: madvise away the ring buffer? RemoveThreadStats(t); t->Destroy(); SpinMutexLock l(&list_mutex_); live_list_.Remove(t); free_list_.Push(t); + DontNeedThread(t); } Thread *GetThreadByBufferAddress(uptr p) { - uptr align = ring_buffer_size_ * 2; - return (Thread *)(RoundDownTo(p, align) - sizeof(Thread)); + return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) + + ring_buffer_size_); } uptr MemoryUsedPerThread() { @@ -176,15 +189,17 @@ class HwasanThreadList { private: Thread *AllocThread() { uptr align = ring_buffer_size_ * 2; - uptr ring_buffer_start = RoundUpTo(free_space_ + sizeof(Thread), align); - free_space_ = ring_buffer_start + ring_buffer_size_; + CHECK(IsAligned(free_space_, align)); + Thread *t = (Thread *)(free_space_ + ring_buffer_size_); + free_space_ += thread_alloc_size_; CHECK(free_space_ <= free_space_end_ && "out of thread memory"); - return (Thread *)(ring_buffer_start - sizeof(Thread)); + return t; } uptr free_space_; uptr free_space_end_; uptr ring_buffer_size_; + uptr thread_alloc_size_; ThreadListHead free_list_; ThreadListHead live_list_; diff --git a/lib/interception/interception.h b/lib/interception/interception.h index 87b2365fd767..dacfa5ede28d 100644 --- a/lib/interception/interception.h +++ b/lib/interception/interception.h @@ -1,9 +1,8 @@ //===-- interception.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -186,11 +185,17 @@ const interpose_substitution substitution_##func_name[] \ #endif // SANITIZER_MAC #if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS -#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ +# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ DECLARE_REAL(ret_type, func, __VA_ARGS__) \ extern "C" ret_type WRAP(func)(__VA_ARGS__); +// Declare an interceptor and its wrapper defined in a different translation +// unit (ex. asm). +# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type WRAP(func)(__VA_ARGS__); \ + extern "C" ret_type func(__VA_ARGS__); #else -#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) +# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) +# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) #endif // Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR diff --git a/lib/interception/interception_linux.cc b/lib/interception/interception_linux.cc index 26bfcd8f6794..4b27102a159c 100644 --- a/lib/interception/interception_linux.cc +++ b/lib/interception/interception_linux.cc @@ -1,9 +1,8 @@ //===-- interception_linux.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,33 +18,62 @@ #include <dlfcn.h> // for dlsym() and dlvsym() +namespace __interception { + #if SANITIZER_NETBSD -#include "sanitizer_common/sanitizer_libc.h" +static int StrCmp(const char *s1, const char *s2) { + while (true) { + if (*s1 != *s2) + return false; + if (*s1 == 0) + return true; + s1++; + s2++; + } +} #endif -namespace __interception { -bool GetRealFunctionAddress(const char *func_name, uptr *func_addr, - uptr real, uptr wrapper) { +static void *GetFuncAddr(const char *name, uptr wrapper_addr) { #if SANITIZER_NETBSD - // XXX: Find a better way to handle renames - if (internal_strcmp(func_name, "sigaction") == 0) func_name = "__sigaction14"; + // FIXME: Find a better way to handle renames + if (StrCmp(name, "sigaction")) + name = "__sigaction14"; #endif - *func_addr = (uptr)dlsym(RTLD_NEXT, func_name); - if (!*func_addr) { + void *addr = dlsym(RTLD_NEXT, name); + if (!addr) { // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is // later in the library search order than the DSO that we are trying to // intercept, which means that we cannot intercept this function. We still // want the address of the real definition, though, so look it up using // RTLD_DEFAULT. - *func_addr = (uptr)dlsym(RTLD_DEFAULT, func_name); + addr = dlsym(RTLD_DEFAULT, name); + + // In case `name' is not loaded, dlsym ends up finding the actual wrapper. + // We don't want to intercept the wrapper and have it point to itself. + if ((uptr)addr == wrapper_addr) + addr = nullptr; } - return real == wrapper; + return addr; +} + +bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, + uptr wrapper) { + void *addr = GetFuncAddr(name, wrapper); + *ptr_to_real = (uptr)addr; + return addr && (func == wrapper); } // Android and Solaris do not have dlvsym #if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD -void *GetFuncAddrVer(const char *func_name, const char *ver) { - return dlvsym(RTLD_NEXT, func_name, ver); +static void *GetFuncAddr(const char *name, const char *ver) { + return dlvsym(RTLD_NEXT, name, ver); +} + +bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, + uptr func, uptr wrapper) { + void *addr = GetFuncAddr(name, ver); + *ptr_to_real = (uptr)addr; + return addr && (func == wrapper); } #endif // !SANITIZER_ANDROID diff --git a/lib/interception/interception_linux.h b/lib/interception/interception_linux.h index 765a186e5823..e578da0cf64e 100644 --- a/lib/interception/interception_linux.h +++ b/lib/interception/interception_linux.h @@ -1,9 +1,8 @@ //===-- interception_linux.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,23 +22,27 @@ #define INTERCEPTION_LINUX_H namespace __interception { -// returns true if a function with the given name was found. -bool GetRealFunctionAddress(const char *func_name, uptr *func_addr, - uptr real, uptr wrapper); -void *GetFuncAddrVer(const char *func_name, const char *ver); +bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, + uptr wrapper); +bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, + uptr func, uptr wrapper); } // namespace __interception -#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ - ::__interception::GetRealFunctionAddress( \ - #func, (::__interception::uptr *)&__interception::PTR_TO_REAL(func), \ - (::__interception::uptr) & (func), \ +#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ + ::__interception::InterceptFunction( \ + #func, \ + (::__interception::uptr *) & REAL(func), \ + (::__interception::uptr) & (func), \ (::__interception::uptr) & WRAP(func)) // Android, Solaris and OpenBSD do not have dlvsym #if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ - (::__interception::real_##func = (func##_type)( \ - unsigned long)::__interception::GetFuncAddrVer(#func, symver)) + ::__interception::InterceptFunction( \ + #func, symver, \ + (::__interception::uptr *) & REAL(func), \ + (::__interception::uptr) & (func), \ + (::__interception::uptr) & WRAP(func)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) diff --git a/lib/interception/interception_mac.cc b/lib/interception/interception_mac.cc index ea8072f8dd73..5bfc1514d2b8 100644 --- a/lib/interception/interception_mac.cc +++ b/lib/interception/interception_mac.cc @@ -1,9 +1,8 @@ //===-- interception_mac.cc -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/interception_mac.h b/lib/interception/interception_mac.h index 6f31fed47b84..eddedb8959c4 100644 --- a/lib/interception/interception_mac.h +++ b/lib/interception/interception_mac.h @@ -1,9 +1,8 @@ //===-- interception_mac.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/interception_type_test.cc b/lib/interception/interception_type_test.cc index 2b3a6d509bfb..c00294a9b474 100644 --- a/lib/interception/interception_type_test.cc +++ b/lib/interception/interception_type_test.cc @@ -1,9 +1,8 @@ //===-- interception_type_test.cc -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/interception/interception_win.cc b/lib/interception/interception_win.cc index cd13827e5857..40bde008052b 100644 --- a/lib/interception/interception_win.cc +++ b/lib/interception/interception_win.cc @@ -1,9 +1,8 @@ //===-- interception_linux.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -513,10 +512,12 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xc0854d: // 4d 85 c0 : test r8, r8 case 0xc2b60f: // 0f b6 c2 : movzx eax, dl case 0xc03345: // 45 33 c0 : xor r8d, r8d + case 0xc93345: // 45 33 c9 : xor r9d, r9d case 0xdb3345: // 45 33 DB : xor r11d, r11d case 0xd98b4c: // 4c 8b d9 : mov r11, rcx case 0xd28b4c: // 4c 8b d2 : mov r10, rdx case 0xc98b4c: // 4C 8B C9 : mov r9, rcx + case 0xc18b4c: // 4C 8B C1 : mov r8, rcx case 0xd2b60f: // 0f b6 d2 : movzx edx, dl case 0xca2b48: // 48 2b ca : sub rcx, rdx case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax] @@ -524,6 +525,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xd18b48: // 48 8b d1 : mov rdx, rcx case 0xdc8b4c: // 4c 8b dc : mov r11, rsp case 0xd18b4c: // 4c 8b d1 : mov r10, rcx + case 0xE0E483: // 83 E4 E0 : and esp, 0xFFFFFFE0 return 3; case 0xec8348: // 48 83 ec XX : sub rsp, XX @@ -555,6 +557,9 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx + case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx + case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9 + case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8 return 5; case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY return 6; diff --git a/lib/interception/interception_win.h b/lib/interception/interception_win.h index 71a44428f5c7..4590013019e3 100644 --- a/lib/interception/interception_win.h +++ b/lib/interception/interception_win.h @@ -1,9 +1,8 @@ //===-- interception_linux.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan.cc b/lib/lsan/lsan.cc index 93bced0459c2..68697a6363e8 100644 --- a/lib/lsan/lsan.cc +++ b/lib/lsan/lsan.cc @@ -1,9 +1,8 @@ //=-- lsan.cc -------------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,6 +32,24 @@ bool WordIsPoisoned(uptr addr) { } // namespace __lsan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __lsan; + uptr stack_top = 0, stack_bottom = 0; + ThreadContext *t; + if (StackTrace::WillUseFastUnwind(request_fast) && + (t = CurrentThreadContext())) { + stack_top = t->stack_end(); + stack_bottom = t->stack_begin(); + } + if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, stack_top, stack_bottom, true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); + } +} + using namespace __lsan; // NOLINT static void InitializeFlags() { @@ -59,7 +76,7 @@ static void InitializeFlags() { // Override from user-specified string. const char *lsan_default_options = MaybeCallLsanDefaultOptions(); parser.ParseString(lsan_default_options); - parser.ParseString(GetEnv("LSAN_OPTIONS")); + parser.ParseStringFromEnv("LSAN_OPTIONS"); SetVerbosity(common_flags()->verbosity); @@ -72,7 +89,7 @@ static void InitializeFlags() { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + stack->Unwind(sig.pc, sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } diff --git a/lib/lsan/lsan.h b/lib/lsan/lsan.h index 6baee81d00cd..9904ada4bb3b 100644 --- a/lib/lsan/lsan.h +++ b/lib/lsan/lsan.h @@ -1,9 +1,8 @@ //=-- lsan.h --------------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,8 @@ #define GET_STACK_TRACE(max_size, fast) \ __sanitizer::BufferedStackTrace stack; \ - GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), nullptr, fast); + stack.Unwind(StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), nullptr, fast, max_size); #define GET_STACK_TRACE_FATAL \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) @@ -41,24 +40,6 @@ void ReplaceSystemMalloc(); __lsan_init(); \ } while (0) -// Get the stack trace with the given pc and bp. -// The pc will be in the position 0 of the resulting stack trace. -// The bp may refer to the current frame or to the caller's frame. -ALWAYS_INLINE -void GetStackTrace(__sanitizer::BufferedStackTrace *stack, - __sanitizer::uptr max_depth, __sanitizer::uptr pc, - __sanitizer::uptr bp, void *context, bool fast) { - uptr stack_top = 0, stack_bottom = 0; - ThreadContext *t; - if (fast && (t = CurrentThreadContext())) { - stack_top = t->stack_end(); - stack_bottom = t->stack_begin(); - } - if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { - stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast); - } -} - } // namespace __lsan extern bool lsan_inited; diff --git a/lib/lsan/lsan_allocator.cc b/lib/lsan/lsan_allocator.cc index 1b338bd5973e..8b13e4c028d4 100644 --- a/lib/lsan/lsan_allocator.cc +++ b/lib/lsan/lsan_allocator.cc @@ -1,9 +1,8 @@ //=-- lsan_allocator.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -186,6 +185,17 @@ void *lsan_realloc(void *p, uptr size, const StackTrace &stack) { return SetErrnoOnNull(Reallocate(stack, p, size, 1)); } +void *lsan_reallocarray(void *ptr, uptr nmemb, uptr size, + const StackTrace &stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, &stack); + } + return lsan_realloc(ptr, nmemb * size, stack); +} + void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) { return SetErrnoOnNull(Calloc(nmemb, size, stack)); } diff --git a/lib/lsan/lsan_allocator.h b/lib/lsan/lsan_allocator.h index 4c4e02fc0902..e13970997672 100644 --- a/lib/lsan/lsan_allocator.h +++ b/lib/lsan/lsan_allocator.h @@ -1,9 +1,8 @@ //=-- lsan_allocator.h ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -52,21 +51,14 @@ struct ChunkMetadata { #if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \ defined(__arm__) -static const uptr kRegionSizeLog = 20; -static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -template <typename AddressSpaceView> -using ByteMapASVT = - TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; - template <typename AddressSpaceViewTy> struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = sizeof(ChunkMetadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __lsan::kRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = AddressSpaceViewTy; - using ByteMap = __lsan::ByteMapASVT<AddressSpaceView>; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -98,23 +90,11 @@ using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #endif template <typename AddressSpaceView> -using AllocatorCacheASVT = - SizeClassAllocatorLocalCache<PrimaryAllocatorASVT<AddressSpaceView>>; -using AllocatorCache = AllocatorCacheASVT<LocalAddressSpaceView>; - -template <typename AddressSpaceView> -using SecondaryAllocatorASVT = - LargeMmapAllocator<NoOpMapUnmapCallback, DefaultLargeMmapAllocatorPtrArray, - AddressSpaceView>; - -template <typename AddressSpaceView> -using AllocatorASVT = - CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>, - AllocatorCacheASVT<AddressSpaceView>, - SecondaryAllocatorASVT<AddressSpaceView>>; +using AllocatorASVT = CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>; using Allocator = AllocatorASVT<LocalAddressSpaceView>; +using AllocatorCache = Allocator::AllocatorCache; -AllocatorCache *GetAllocatorCache(); +Allocator::AllocatorCache *GetAllocatorCache(); int lsan_posix_memalign(void **memptr, uptr alignment, uptr size, const StackTrace &stack); @@ -123,6 +103,8 @@ void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack); void *lsan_malloc(uptr size, const StackTrace &stack); void lsan_free(void *p); void *lsan_realloc(void *p, uptr size, const StackTrace &stack); +void *lsan_reallocarray(void *p, uptr nmemb, uptr size, + const StackTrace &stack); void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack); void *lsan_valloc(uptr size, const StackTrace &stack); void *lsan_pvalloc(uptr size, const StackTrace &stack); diff --git a/lib/lsan/lsan_common.cc b/lib/lsan/lsan_common.cc index eaa5cadc8ffb..7c842a152d54 100644 --- a/lib/lsan/lsan_common.cc +++ b/lib/lsan/lsan_common.cc @@ -1,9 +1,8 @@ //=-- lsan_common.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_common.h b/lib/lsan/lsan_common.h index 1d1e1e462435..58dc00faaee5 100644 --- a/lib/lsan/lsan_common.h +++ b/lib/lsan/lsan_common.h @@ -1,9 +1,8 @@ //=-- lsan_common.h -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,8 +21,8 @@ #include "sanitizer_common/sanitizer_stoptheworld.h" #include "sanitizer_common/sanitizer_symbolizer.h" -// LeakSanitizer relies on some Glibc's internals (e.g. TLS machinery) thus -// supported for Linux only. Also, LSan doesn't like 32 bit architectures +// LeakSanitizer relies on some Glibc's internals (e.g. TLS machinery) on Linux. +// Also, LSan doesn't like 32 bit architectures // because of "small" (4 bytes) pointer size that leads to high false negative // ratio on large leaks. But we still want to have it for some 32 bit arches // (e.g. x86), see https://github.com/google/sanitizers/issues/403. @@ -41,6 +40,8 @@ #elif defined(__arm__) && \ SANITIZER_LINUX && !SANITIZER_ANDROID #define CAN_SANITIZE_LEAKS 1 +#elif SANITIZER_NETBSD +#define CAN_SANITIZE_LEAKS 1 #else #define CAN_SANITIZE_LEAKS 0 #endif diff --git a/lib/lsan/lsan_common_linux.cc b/lib/lsan/lsan_common_linux.cc index cdd7f032a192..ef4f591d88f7 100644 --- a/lib/lsan/lsan_common_linux.cc +++ b/lib/lsan/lsan_common_linux.cc @@ -1,21 +1,21 @@ //=-- lsan_common_linux.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of LeakSanitizer. -// Implementation of common leak checking functionality. Linux-specific code. +// Implementation of common leak checking functionality. Linux/NetBSD-specific +// code. // //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" #include "lsan_common.h" -#if CAN_SANITIZE_LEAKS && SANITIZER_LINUX +#if CAN_SANITIZE_LEAKS && (SANITIZER_LINUX || SANITIZER_NETBSD) #include <link.h> #include "sanitizer_common/sanitizer_common.h" @@ -137,4 +137,4 @@ void DoStopTheWorld(StopTheWorldCallback callback, void *argument) { } // namespace __lsan -#endif // CAN_SANITIZE_LEAKS && SANITIZER_LINUX +#endif diff --git a/lib/lsan/lsan_common_mac.cc b/lib/lsan/lsan_common_mac.cc index a355cea96c43..14c2b3711994 100644 --- a/lib/lsan/lsan_common_mac.cc +++ b/lib/lsan/lsan_common_mac.cc @@ -1,9 +1,8 @@ //=-- lsan_common_mac.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_flags.inc b/lib/lsan/lsan_flags.inc index e390e2ae5a1b..9350f4bcdc34 100644 --- a/lib/lsan/lsan_flags.inc +++ b/lib/lsan/lsan_flags.inc @@ -1,9 +1,8 @@ //===-- lsan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_interceptors.cc b/lib/lsan/lsan_interceptors.cc index a9bd2ba42319..4a4c86a9dca0 100644 --- a/lib/lsan/lsan_interceptors.cc +++ b/lib/lsan/lsan_interceptors.cc @@ -1,9 +1,8 @@ //=-- lsan_interceptors.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -84,6 +83,12 @@ INTERCEPTOR(void*, realloc, void *q, uptr size) { return lsan_realloc(q, size, stack); } +INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) { + ENSURE_LSAN_INITED; + GET_STACK_TRACE_MALLOC; + return lsan_reallocarray(q, nmemb, size, stack); +} + INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) { ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; diff --git a/lib/lsan/lsan_linux.cc b/lib/lsan/lsan_linux.cc index c9749c745655..22d034280d7d 100644 --- a/lib/lsan/lsan_linux.cc +++ b/lib/lsan/lsan_linux.cc @@ -1,19 +1,18 @@ //=-- lsan_linux.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // -// This file is a part of LeakSanitizer. Linux-specific code. +// This file is a part of LeakSanitizer. Linux/NetBSD-specific code. // //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_NETBSD #include "lsan_allocator.h" @@ -30,4 +29,4 @@ void ReplaceSystemMalloc() {} } // namespace __lsan -#endif // SANITIZER_LINUX +#endif // SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/lib/lsan/lsan_mac.cc b/lib/lsan/lsan_mac.cc index 1a6f5f4899b1..435f41b6f8bc 100644 --- a/lib/lsan/lsan_mac.cc +++ b/lib/lsan/lsan_mac.cc @@ -1,9 +1,8 @@ //===-- lsan_mac.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_malloc_mac.cc b/lib/lsan/lsan_malloc_mac.cc index 94ffb6d02539..34447b4b39fc 100644 --- a/lib/lsan/lsan_malloc_mac.cc +++ b/lib/lsan/lsan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- lsan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -52,6 +51,8 @@ using namespace __lsan; (void)zone_name; \ Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); #define COMMON_MALLOC_NAMESPACE __lsan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0 #include "sanitizer_common/sanitizer_malloc_mac.inc" diff --git a/lib/lsan/lsan_preinit.cc b/lib/lsan/lsan_preinit.cc index 5a190959c15d..5d0ad89a8b02 100644 --- a/lib/lsan/lsan_preinit.cc +++ b/lib/lsan/lsan_preinit.cc @@ -1,9 +1,8 @@ //===-- lsan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/lsan/lsan_thread.cc b/lib/lsan/lsan_thread.cc index a25aff379961..77f6a9236dde 100644 --- a/lib/lsan/lsan_thread.cc +++ b/lib/lsan/lsan_thread.cc @@ -1,9 +1,8 @@ //=-- lsan_thread.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -77,7 +76,7 @@ u32 ThreadCreate(u32 parent_tid, uptr user_id, bool detached) { /* arg */ nullptr); } -void ThreadStart(u32 tid, tid_t os_id, bool workerthread) { +void ThreadStart(u32 tid, tid_t os_id, ThreadType thread_type) { OnStartedArgs args; uptr stack_size = 0; uptr tls_size = 0; @@ -87,7 +86,7 @@ void ThreadStart(u32 tid, tid_t os_id, bool workerthread) { args.tls_end = args.tls_begin + tls_size; GetAllocatorCacheRange(&args.cache_begin, &args.cache_end); args.dtls = DTLS_Get(); - thread_registry->StartThread(tid, os_id, workerthread, &args); + thread_registry->StartThread(tid, os_id, thread_type, &args); } void ThreadFinish() { diff --git a/lib/lsan/lsan_thread.h b/lib/lsan/lsan_thread.h index b16d3d91537f..b869d066d9d8 100644 --- a/lib/lsan/lsan_thread.h +++ b/lib/lsan/lsan_thread.h @@ -1,9 +1,8 @@ //=-- lsan_thread.h -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,7 +44,8 @@ class ThreadContext : public ThreadContextBase { void InitializeThreadRegistry(); -void ThreadStart(u32 tid, tid_t os_id, bool workerthread = false); +void ThreadStart(u32 tid, tid_t os_id, + ThreadType thread_type = ThreadType::Regular); void ThreadFinish(); u32 ThreadCreate(u32 tid, uptr uid, bool detached); void ThreadJoin(u32 tid); diff --git a/lib/msan/msan.cc b/lib/msan/msan.cc index ba2d5d593303..c62e5cd4c518 100644 --- a/lib/msan/msan.cc +++ b/lib/msan/msan.cc @@ -1,9 +1,8 @@ //===-- msan.cc -----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -176,12 +175,10 @@ static void InitializeFlags() { ubsan_parser.ParseString(ubsan_default_options); #endif - const char *msan_options = GetEnv("MSAN_OPTIONS"); - parser.ParseString(msan_options); + parser.ParseStringFromEnv("MSAN_OPTIONS"); #if MSAN_CONTAINS_UBSAN - ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); #endif - VPrintf(1, "MSAN_OPTIONS: %s\n", msan_options ? msan_options : "<empty>"); InitializeCommonFlags(); @@ -222,18 +219,6 @@ static void InitializeFlags() { if (f->store_context_size < 1) f->store_context_size = 1; } -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind) { - MsanThread *t = GetCurrentThread(); - if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { - // Block reports from our interceptors during _Unwind_Backtrace. - SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); - } - stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), - request_fast_unwind); -} - void PrintWarning(uptr pc, uptr bp) { PrintWarningWithOrigin(pc, bp, __msan_origin_tls); } @@ -314,6 +299,21 @@ u32 ChainOrigin(u32 id, StackTrace *stack) { } // namespace __msan +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __msan; + MsanThread *t = GetCurrentThread(); + if (!t || !StackTrace::WillUseFastUnwind(request_fast)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return Unwind(max_depth, pc, bp, context, 0, 0, false); + } + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} + // Interface. using namespace __msan; @@ -378,7 +378,7 @@ void __msan_warning_noreturn() { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + stack->Unwind(sig.pc, sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } diff --git a/lib/msan/msan.h b/lib/msan/msan.h index 4c2e9f9e24a0..ac5f67e6ab3d 100644 --- a/lib/msan/msan.h +++ b/lib/msan/msan.h @@ -1,9 +1,8 @@ //===-- msan.h --------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -289,6 +288,7 @@ void MsanDeallocate(StackTrace *stack, void *ptr); void *msan_malloc(uptr size, StackTrace *stack); void *msan_calloc(uptr nmemb, uptr size, StackTrace *stack); void *msan_realloc(void *ptr, uptr size, StackTrace *stack); +void *msan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack); void *msan_valloc(uptr size, StackTrace *stack); void *msan_pvalloc(uptr size, StackTrace *stack); void *msan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); @@ -313,9 +313,6 @@ struct SymbolizerScope { void PrintWarning(uptr pc, uptr bp); void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin); -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind); - // Unpoison first n function arguments. void UnpoisonParam(uptr n); void UnpoisonThreadLocalState(); @@ -329,23 +326,21 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_MALLOC_STACK_TRACE \ BufferedStackTrace stack; \ if (__msan_get_track_origins() && msan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ - common_flags()->fast_unwind_on_malloc) + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ + nullptr, common_flags()->fast_unwind_on_malloc, \ + common_flags()->malloc_context_size) // For platforms which support slow unwinder only, we restrict the store context // size to 1, basically only storing the current pc. We do this because the slow // unwinder which is based on libunwind is not async signal safe and causes // random freezes in forking applications as well as in signal handlers. -#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ - BufferedStackTrace stack; \ - if (__msan_get_track_origins() > 1 && msan_inited) { \ - if (!SANITIZER_CAN_FAST_UNWIND) \ - GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \ - nullptr, false); \ - else \ - GetStackTrace(&stack, flags()->store_context_size, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_malloc); \ +#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (__msan_get_track_origins() > 1 && msan_inited) { \ + int size = flags()->store_context_size; \ + if (!SANITIZER_CAN_FAST_UNWIND) \ + size = Min(size, 1); \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_malloc, size);\ } #define GET_STORE_STACK_TRACE \ @@ -354,8 +349,7 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ BufferedStackTrace stack; \ if (msan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_FATAL_STACK_TRACE_HERE \ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()) diff --git a/lib/msan/msan_allocator.cc b/lib/msan/msan_allocator.cc index 053ab0280397..1816840012e4 100644 --- a/lib/msan/msan_allocator.cc +++ b/lib/msan/msan_allocator.cc @@ -1,9 +1,8 @@ //===-- msan_allocator.cc --------------------------- ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,81 +45,71 @@ struct MsanMapUnmapCallback { }; #if defined(__mips64) - static const uptr kMaxAllowedMallocSize = 2UL << 30; +static const uptr kMaxAllowedMallocSize = 2UL << 30; + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = 20; - static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; - typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; - - struct AP32 { - static const uptr kSpaceBeg = 0; - static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; - static const uptr kMetadataSize = sizeof(Metadata); - typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __msan::ByteMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - }; - typedef SizeClassAllocator32<AP32> PrimaryAllocator; + using AddressSpaceView = LocalAddressSpaceView; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32<AP32> PrimaryAllocator; #elif defined(__x86_64__) #if SANITIZER_NETBSD || \ (SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING)) - static const uptr kAllocatorSpace = 0x700000000000ULL; +static const uptr kAllocatorSpace = 0x700000000000ULL; #else - static const uptr kAllocatorSpace = 0x600000000000ULL; +static const uptr kAllocatorSpace = 0x600000000000ULL; #endif - static const uptr kMaxAllowedMallocSize = 8UL << 30; - - struct AP64 { // Allocator64 parameters. Deliberately using a short name. - static const uptr kSpaceBeg = kAllocatorSpace; - static const uptr kSpaceSize = 0x40000000000; // 4T. - static const uptr kMetadataSize = sizeof(Metadata); - typedef DefaultSizeClassMap SizeClassMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - using AddressSpaceView = LocalAddressSpaceView; - }; +static const uptr kMaxAllowedMallocSize = 8UL << 30; + +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = 0x40000000000; // 4T. + static const uptr kMetadataSize = sizeof(Metadata); + typedef DefaultSizeClassMap SizeClassMap; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; +}; - typedef SizeClassAllocator64<AP64> PrimaryAllocator; +typedef SizeClassAllocator64<AP64> PrimaryAllocator; #elif defined(__powerpc64__) - static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G - - struct AP64 { // Allocator64 parameters. Deliberately using a short name. - static const uptr kSpaceBeg = 0x300000000000; - static const uptr kSpaceSize = 0x020000000000; // 2T. - static const uptr kMetadataSize = sizeof(Metadata); - typedef DefaultSizeClassMap SizeClassMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - using AddressSpaceView = LocalAddressSpaceView; - }; - - typedef SizeClassAllocator64<AP64> PrimaryAllocator; +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = 0x300000000000; + static const uptr kSpaceSize = 0x020000000000; // 2T. + static const uptr kMetadataSize = sizeof(Metadata); + typedef DefaultSizeClassMap SizeClassMap; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; +}; + +typedef SizeClassAllocator64<AP64> PrimaryAllocator; #elif defined(__aarch64__) - static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = 20; - static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; - typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; - - struct AP32 { - static const uptr kSpaceBeg = 0; - static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; - static const uptr kMetadataSize = sizeof(Metadata); - typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __msan::ByteMap; - typedef MsanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; - }; - typedef SizeClassAllocator32<AP32> PrimaryAllocator; + using AddressSpaceView = LocalAddressSpaceView; + typedef MsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32<AP32> PrimaryAllocator; #endif -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<MsanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; static Allocator allocator; static AllocatorCache fallback_allocator_cache; @@ -270,6 +259,16 @@ void *msan_realloc(void *ptr, uptr size, StackTrace *stack) { return SetErrnoOnNull(MsanReallocate(stack, ptr, size, sizeof(u64))); } +void *msan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return msan_realloc(ptr, nmemb * size, stack); +} + void *msan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull(MsanAllocate(stack, size, GetPageSizeCached(), false)); } diff --git a/lib/msan/msan_allocator.h b/lib/msan/msan_allocator.h index 407942e54c1a..42a5022c9e3e 100644 --- a/lib/msan/msan_allocator.h +++ b/lib/msan/msan_allocator.h @@ -1,9 +1,8 @@ //===-- msan_allocator.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_chained_origin_depot.cc b/lib/msan/msan_chained_origin_depot.cc index e2796fd46464..6c634252e0b1 100644 --- a/lib/msan/msan_chained_origin_depot.cc +++ b/lib/msan/msan_chained_origin_depot.cc @@ -1,9 +1,8 @@ //===-- msan_chained_origin_depot.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_chained_origin_depot.h b/lib/msan/msan_chained_origin_depot.h index f7a71cef205f..2b4cb3647791 100644 --- a/lib/msan/msan_chained_origin_depot.h +++ b/lib/msan/msan_chained_origin_depot.h @@ -1,9 +1,8 @@ //===-- msan_chained_origin_depot.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_flags.h b/lib/msan/msan_flags.h index 4fc6d172a04a..836dbbaa164a 100644 --- a/lib/msan/msan_flags.h +++ b/lib/msan/msan_flags.h @@ -1,9 +1,8 @@ //===-- msan_flags.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_flags.inc b/lib/msan/msan_flags.inc index a7ff6c586071..e6a26015a22a 100644 --- a/lib/msan/msan_flags.inc +++ b/lib/msan/msan_flags.inc @@ -1,9 +1,8 @@ //===-- msan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_interceptors.cc b/lib/msan/msan_interceptors.cc index 497f943a8a0e..b055bb749cb8 100644 --- a/lib/msan/msan_interceptors.cc +++ b/lib/msan/msan_interceptors.cc @@ -1,9 +1,8 @@ //===-- msan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -908,6 +907,11 @@ INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { return msan_realloc(ptr, size, &stack); } +INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return msan_reallocarray(ptr, nmemb, size, &stack); +} + INTERCEPTOR(void *, malloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!msan_inited)) @@ -1119,8 +1123,12 @@ void MSanAtExitWrapper() { void MSanCxaAtExitWrapper(void *arg) { UnpoisonParam(1); MSanAtExitRecord *r = (MSanAtExitRecord *)arg; + // libc before 2.27 had race which caused occasional double handler execution + // https://sourceware.org/ml/libc-alpha/2017-08/msg01204.html + if (!r->func) + return; r->func(r->arg); - InternalFree(r); + r->func = nullptr; } static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso); @@ -1184,14 +1192,14 @@ INTERCEPTOR(int, fork, void) { // NetBSD ships with openpty(3) in -lutil, that needs to be prebuilt explicitly // with MSan. #if SANITIZER_LINUX -INTERCEPTOR(int, openpty, int *amaster, int *aslave, char *name, +INTERCEPTOR(int, openpty, int *aparent, int *aworker, char *name, const void *termp, const void *winp) { ENSURE_MSAN_INITED(); InterceptorScope interceptor_scope; - int res = REAL(openpty)(amaster, aslave, name, termp, winp); + int res = REAL(openpty)(aparent, aworker, name, termp, winp); if (!res) { - __msan_unpoison(amaster, sizeof(*amaster)); - __msan_unpoison(aslave, sizeof(*aslave)); + __msan_unpoison(aparent, sizeof(*aparent)); + __msan_unpoison(aworker, sizeof(*aworker)); } return res; } @@ -1203,13 +1211,13 @@ INTERCEPTOR(int, openpty, int *amaster, int *aslave, char *name, // NetBSD ships with forkpty(3) in -lutil, that needs to be prebuilt explicitly // with MSan. #if SANITIZER_LINUX -INTERCEPTOR(int, forkpty, int *amaster, char *name, const void *termp, +INTERCEPTOR(int, forkpty, int *aparent, char *name, const void *termp, const void *winp) { ENSURE_MSAN_INITED(); InterceptorScope interceptor_scope; - int res = REAL(forkpty)(amaster, name, termp, winp); + int res = REAL(forkpty)(aparent, name, termp, winp); if (res != -1) - __msan_unpoison(amaster, sizeof(*amaster)); + __msan_unpoison(aparent, sizeof(*aparent)); return res; } #define MSAN_MAYBE_INTERCEPT_FORKPTY INTERCEPT_FUNCTION(forkpty) @@ -1238,17 +1246,17 @@ int OnExit() { CHECK_UNPOISONED_0(x, n); \ } while (0) -#define MSAN_INTERCEPT_FUNC(name) \ - do { \ - if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ - VReport(1, "MemorySanitizer: failed to intercept '" #name "'\n"); \ +#define MSAN_INTERCEPT_FUNC(name) \ + do { \ + if (!INTERCEPT_FUNCTION(name)) \ + VReport(1, "MemorySanitizer: failed to intercept '%s'\n'", #name); \ } while (0) -#define MSAN_INTERCEPT_FUNC_VER(name, ver) \ - do { \ - if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ - VReport( \ - 1, "MemorySanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ +#define MSAN_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if (!INTERCEPT_FUNCTION_VER(name, ver)) \ + VReport(1, "MemorySanitizer: failed to intercept '%s@@%s'\n", #name, \ + #ver); \ } while (0) #define COMMON_INTERCEPT_FUNCTION(name) MSAN_INTERCEPT_FUNC(name) @@ -1535,6 +1543,8 @@ void __msan_poison_stack(void *a, uptr size) { SetShadow(a, size, __msan::flags()->poison_stack_with_zeroes ? 0 : -1); } +void __msan_unpoison_param(uptr n) { UnpoisonParam(n); } + void __msan_clear_and_unpoison(void *a, uptr size) { REAL(memset)(a, 0, size); SetShadow(a, size, 0); @@ -1594,6 +1604,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(malloc); INTERCEPT_FUNCTION(calloc); INTERCEPT_FUNCTION(realloc); + INTERCEPT_FUNCTION(reallocarray); INTERCEPT_FUNCTION(free); MSAN_MAYBE_INTERCEPT_CFREE; MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE; diff --git a/lib/msan/msan_interface_internal.h b/lib/msan/msan_interface_internal.h index 9a67cbc9b5f8..1abbba018c92 100644 --- a/lib/msan/msan_interface_internal.h +++ b/lib/msan/msan_interface_internal.h @@ -1,9 +1,8 @@ //===-- msan_interface_internal.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -70,6 +69,8 @@ void __msan_unpoison(const void *a, uptr size); SANITIZER_INTERFACE_ATTRIBUTE void __msan_unpoison_string(const char *s); SANITIZER_INTERFACE_ATTRIBUTE +void __msan_unpoison_param(uptr n); +SANITIZER_INTERFACE_ATTRIBUTE void __msan_clear_and_unpoison(void *a, uptr size); SANITIZER_INTERFACE_ATTRIBUTE void* __msan_memcpy(void *dst, const void *src, uptr size); diff --git a/lib/msan/msan_linux.cc b/lib/msan/msan_linux.cc index 0b0208884d2f..3b6e6cb85f33 100644 --- a/lib/msan/msan_linux.cc +++ b/lib/msan/msan_linux.cc @@ -1,9 +1,8 @@ //===-- msan_linux.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_new_delete.cc b/lib/msan/msan_new_delete.cc index a0959aec5eb0..750981eb55eb 100644 --- a/lib/msan/msan_new_delete.cc +++ b/lib/msan/msan_new_delete.cc @@ -1,9 +1,8 @@ //===-- msan_new_delete.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_origin.h b/lib/msan/msan_origin.h index 36c168b8521c..26a4e7eb90cd 100644 --- a/lib/msan/msan_origin.h +++ b/lib/msan/msan_origin.h @@ -1,9 +1,8 @@ //===-- msan_origin.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_poisoning.cc b/lib/msan/msan_poisoning.cc index 7420d946928b..5ea01f51a835 100644 --- a/lib/msan/msan_poisoning.cc +++ b/lib/msan/msan_poisoning.cc @@ -1,9 +1,8 @@ //===-- msan_poisoning.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_poisoning.h b/lib/msan/msan_poisoning.h index edacbeeab0a6..270f1e24991b 100644 --- a/lib/msan/msan_poisoning.h +++ b/lib/msan/msan_poisoning.h @@ -1,9 +1,8 @@ //===-- msan_poisoning.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_report.cc b/lib/msan/msan_report.cc index 2f0cc8d370ee..73bce3972593 100644 --- a/lib/msan/msan_report.cc +++ b/lib/msan/msan_report.cc @@ -1,9 +1,8 @@ //===-- msan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/msan/msan_report.h b/lib/msan/msan_report.h index 73840e417505..0965b8cb6813 100644 --- a/lib/msan/msan_report.h +++ b/lib/msan/msan_report.h @@ -1,9 +1,8 @@ //===-- msan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/msan/msan_thread.h b/lib/msan/msan_thread.h index ed22e67edd50..808780cd57b9 100644 --- a/lib/msan/msan_thread.h +++ b/lib/msan/msan_thread.h @@ -1,9 +1,8 @@ //===-- msan_thread.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/profile/GCDAProfiling.c b/lib/profile/GCDAProfiling.c index 0665a680cf01..498c05900bf2 100644 --- a/lib/profile/GCDAProfiling.c +++ b/lib/profile/GCDAProfiling.c @@ -1,9 +1,8 @@ /*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------===*| |* @@ -269,14 +268,14 @@ static int map_file() { mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL); if (mmap_handle == NULL) { - fprintf(stderr, "profiling: %s: cannot create file mapping: %d\n", filename, - GetLastError()); + fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n", + filename, GetLastError()); return -1; } write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size); if (write_buffer == NULL) { - fprintf(stderr, "profiling: %s: cannot map: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename, GetLastError()); CloseHandle(mmap_handle); return -1; @@ -298,18 +297,18 @@ static int map_file() { static void unmap_file() { #if defined(_WIN32) if (!FlushViewOfFile(write_buffer, file_size)) { - fprintf(stderr, "profiling: %s: cannot flush mapped view: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename, GetLastError()); } if (!UnmapViewOfFile(write_buffer)) { - fprintf(stderr, "profiling: %s: cannot unmap mapped view: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename, GetLastError()); } if (!CloseHandle(mmap_handle)) { - fprintf(stderr, "profiling: %s: cannot close file mapping handle: %d\n", filename, - GetLastError()); + fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n", + filename, GetLastError()); } mmap_handle = NULL; diff --git a/lib/profile/InstrProfData.inc b/lib/profile/InstrProfData.inc index 454620ed997a..7078af5f4cf8 100644 --- a/lib/profile/InstrProfData.inc +++ b/lib/profile/InstrProfData.inc @@ -1,9 +1,8 @@ /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -170,7 +169,7 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) /* VALUE_PROF_KIND start */ #ifndef VALUE_PROF_KIND -#define VALUE_PROF_KIND(Enumerator, Value) +#define VALUE_PROF_KIND(Enumerator, Value, Descr) #else #define INSTR_PROF_DATA_DEFINED #endif @@ -183,16 +182,16 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) * For this remapping the ProfData is used. ProfData contains both the function * name hash and the function address. */ -VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ -VALUE_PROF_KIND(IPVK_MemOPSize, 1) +VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ -VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize) +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -250,22 +249,25 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_COFF), "__DATA,") + INSTR_PROF_DATA_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COFF), "__DATA,") + INSTR_PROF_CNTS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_COFF), "__DATA,") + INSTR_PROF_NAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_COFF), "__DATA,") + INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COFF), "__DATA,") + INSTR_PROF_VNODES_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COFF), "__LLVM_COV,") + INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -636,10 +638,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 57 indicates there are context-sensitive records in the profile. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime @@ -655,13 +659,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap -/* Win32 */ -#define INSTR_PROF_DATA_COFF .lprfd -#define INSTR_PROF_NAME_COFF .lprfn -#define INSTR_PROF_CNTS_COFF .lprfc -#define INSTR_PROF_VALS_COFF .lprfv -#define INSTR_PROF_VNODES_COFF .lprfnd -#define INSTR_PROF_COVMAP_COFF .lcovmap +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +/* Windows section names. Because these section names contain dollar characters, + * they must be quoted. + */ +#define INSTR_PROF_DATA_COFF ".lprfd$M" +#define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_VALS_COFF ".lprfv$M" +#define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -675,32 +683,30 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Value profile nodes section. */ #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ -#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COMMON -#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COMMON -#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COMMON +#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) +#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) +#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ -#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COMMON +#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) /* Value profile nodes section. */ -#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COMMON -#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COMMON +#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) +#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) +/* Order file instrumentation. */ +#define INSTR_PROF_ORDERFILE_SECT_NAME \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) #endif -#define INSTR_PROF_DATA_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) -#define INSTR_PROF_NAME_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) -#define INSTR_PROF_CNTS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) -#define INSTR_PROF_COVMAP_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) -#define INSTR_PROF_VALS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME) -#define INSTR_PROF_VNODES_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer +#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) /* Macros to define start/stop section symbol for a given * section on Linux. For instance @@ -735,6 +741,12 @@ typedef struct InstrProfValueData { #endif /* INSTR_PROF_DATA_INC */ +#ifndef INSTR_ORDER_FILE_INC +/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ +#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 +#define INSTR_ORDER_FILE_BUFFER_BITS 17 +#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff +#endif /* INSTR_ORDER_FILE_INC */ #else #undef INSTR_PROF_DATA_DEFINED #endif diff --git a/lib/profile/InstrProfiling.c b/lib/profile/InstrProfiling.c index 00b31e1ee7d4..f3787715e709 100644 --- a/lib/profile/InstrProfiling.c +++ b/lib/profile/InstrProfiling.c @@ -1,9 +1,8 @@ /*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -36,6 +35,10 @@ COMPILER_RT_VISIBILITY void lprofSetProfileDumped() { ProfileDumped = 1; } +COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped() { + lprofSetProfileDumped(); +} + /* Return the number of bytes needed to add to SizeInBytes to make it * the result a multiple of 8. */ diff --git a/lib/profile/InstrProfiling.h b/lib/profile/InstrProfiling.h index 3ef7eacf06be..4d196a89b492 100644 --- a/lib/profile/InstrProfiling.h +++ b/lib/profile/InstrProfiling.h @@ -1,9 +1,8 @@ /*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -11,12 +10,13 @@ #define PROFILE_INSTRPROFILING_H_ #include "InstrProfilingPort.h" +#include <stdio.h> #define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY #include "InstrProfData.inc" enum ValueKind { -#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, #include "InstrProfData.inc" }; @@ -65,6 +65,7 @@ uint64_t *__llvm_profile_begin_counters(void); uint64_t *__llvm_profile_end_counters(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); +uint32_t *__llvm_profile_begin_orderfile(); /*! * \brief Clear profile counters to zero. @@ -121,10 +122,11 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, */ int __llvm_profile_write_file(void); +int __llvm_orderfile_write_file(void); /*! * \brief this is a wrapper interface to \c __llvm_profile_write_file. * After this interface is invoked, a arleady dumped flag will be set - * so that profile won't be dumped again during program exit. + * so that profile won't be dumped again during program exit. * Invocation of interface __llvm_profile_reset_counters will clear * the flag. This interface is designed to be used to collect profile * data from user selected hot regions. The use model is @@ -143,6 +145,8 @@ int __llvm_profile_write_file(void); */ int __llvm_profile_dump(void); +int __llvm_orderfile_dump(void); + /*! * \brief Set the filename for writing instrumentation data. * @@ -154,6 +158,24 @@ int __llvm_profile_dump(void); */ void __llvm_profile_set_filename(const char *Name); +/*! + * \brief Set the FILE object for writing instrumentation data. + * + * Sets the FILE object to be used for subsequent calls to + * \a __llvm_profile_write_file(). The profile file name set by environment + * variable, command-line option, or calls to \a __llvm_profile_set_filename + * will be ignored. + * + * \c File will not be closed after a call to \a __llvm_profile_write_file() but + * it may be flushed. Passing NULL restores default behavior. + * + * If \c EnableMerge is nonzero, the runtime will always merge profiling data + * with the contents of the profiling file. If EnableMerge is zero, the runtime + * may still merge the data if it would have merged for another reason (for + * example, because of a %m specifier in the file name). + */ +void __llvm_profile_set_file_object(FILE *File, int EnableMerge); + /*! \brief Register to write instrumentation data to file at exit. */ int __llvm_profile_register_write_file_atexit(void); @@ -189,6 +211,14 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End); /*! + * \brief Set the flag that profile data has been dumped to the file. + * This is useful for users to disable dumping profile data to the file for + * certain processes in case the processes don't have permission to write to + * the disks, and trying to do so would result in side effects such as crashes. + */ +void __llvm_profile_set_dumped(); + +/*! * This variable is defined in InstrProfilingRuntime.cc as a hidden * symbol. Its main purpose is to enable profile runtime user to * bypass runtime initialization code -- if the client code explicitly diff --git a/lib/profile/InstrProfilingBuffer.c b/lib/profile/InstrProfilingBuffer.c index a7e852f53ec3..5bdeb8e32807 100644 --- a/lib/profile/InstrProfilingBuffer.c +++ b/lib/profile/InstrProfilingBuffer.c @@ -1,9 +1,8 @@ /*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingFile.c b/lib/profile/InstrProfilingFile.c index c4cf3ccd70c2..e7996e282894 100644 --- a/lib/profile/InstrProfilingFile.c +++ b/lib/profile/InstrProfilingFile.c @@ -1,9 +1,8 @@ /*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -38,7 +37,7 @@ /* From where is profile name specified. * The order the enumerators define their * precedence. Re-order them may lead to - * runtime behavior change. */ + * runtime behavior change. */ typedef enum ProfileNameSpecifier { PNS_unknown = 0, PNS_default, @@ -90,9 +89,27 @@ typedef struct lprofFilename { COMPILER_RT_WEAK lprofFilename lprofCurFilename = {0, 0, 0, 0, {0}, {0}, 0, 0, 0, PNS_unknown}; +static int ProfileMergeRequested = 0; +static int isProfileMergeRequested() { return ProfileMergeRequested; } +static void setProfileMergeRequested(int EnableMerge) { + ProfileMergeRequested = EnableMerge; +} + +static FILE *ProfileFile = NULL; +static FILE *getProfileFile() { return ProfileFile; } +static void setProfileFile(FILE *File) { ProfileFile = File; } + +COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, + int EnableMerge) { + setProfileFile(File); + setProfileMergeRequested(EnableMerge); +} + static int getCurFilenameLength(); static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf); -static unsigned doMerging() { return lprofCurFilename.MergePoolSize; } +static unsigned doMerging() { + return lprofCurFilename.MergePoolSize || isProfileMergeRequested(); +} /* Return 1 if there is an error, otherwise return 0. */ static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, @@ -112,6 +129,15 @@ static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, return 0; } +/* TODO: make buffer size controllable by an internal option, and compiler can pass the size + to runtime via a variable. */ +static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) { + if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) != + INSTR_ORDER_FILE_BUFFER_SIZE) + return 1; + return 0; +} + static void initFileWriter(ProfDataWriter *This, FILE *File) { This->Write = fileWriter; This->WriterCtx = File; @@ -217,11 +243,16 @@ static void createProfileDir(const char *Filename) { * its instrumented shared libraries dump profile data into their own data file. */ static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) { - FILE *ProfileFile; + FILE *ProfileFile = NULL; int rc; - createProfileDir(ProfileFileName); - ProfileFile = lprofOpenFileEx(ProfileFileName); + ProfileFile = getProfileFile(); + if (ProfileFile) { + lprofLockFileHandle(ProfileFile); + } else { + createProfileDir(ProfileFileName); + ProfileFile = lprofOpenFileEx(ProfileFileName); + } if (!ProfileFile) return NULL; @@ -236,6 +267,16 @@ static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) { return ProfileFile; } +static FILE *getFileObject(const char *OutputName) { + FILE *File; + File = getProfileFile(); + if (File != NULL) { + return File; + } + + return fopen(OutputName, "ab"); +} + /* Write profile data to file \c OutputName. */ static int writeFile(const char *OutputName) { int RetVal; @@ -243,10 +284,10 @@ static int writeFile(const char *OutputName) { int MergeDone = 0; VPMergeHook = &lprofMergeValueProfData; - if (!doMerging()) - OutputFile = fopen(OutputName, "ab"); - else + if (doMerging()) OutputFile = openFileForMerging(OutputName, &MergeDone); + else + OutputFile = getFileObject(OutputName); if (!OutputFile) return -1; @@ -257,6 +298,35 @@ static int writeFile(const char *OutputName) { initFileWriter(&fileWriter, OutputFile); RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone); + if (OutputFile == getProfileFile()) { + fflush(OutputFile); + if (doMerging()) { + lprofUnlockFileHandle(OutputFile); + } + } else { + fclose(OutputFile); + } + + return RetVal; +} + +/* Write order data to file \c OutputName. */ +static int writeOrderFile(const char *OutputName) { + int RetVal; + FILE *OutputFile; + + OutputFile = fopen(OutputName, "w"); + + if (!OutputFile) { + PROF_WARN("can't open file with mode ab: %s\n", OutputName); + return -1; + } + + FreeHook = &free; + setupIOBuffer(); + const uint32_t *DataBegin = __llvm_profile_begin_orderfile(); + RetVal = orderFileWriter(OutputFile, DataBegin); + fclose(OutputFile); return RetVal; } @@ -562,7 +632,7 @@ void __llvm_profile_initialize_file(void) { EnvFilenamePat = getFilenamePatFromEnv(); if (EnvFilenamePat) { - /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid + /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid at the moment when __llvm_profile_write_file() gets executed. */ parseAndSetFilename(EnvFilenamePat, PNS_environment, 1); return; @@ -598,8 +668,7 @@ int __llvm_profile_write_file(void) { int PDeathSig = 0; if (lprofProfileDumped()) { - PROF_NOTE("Profile data not written to file: %s.\n", - "already written"); + PROF_NOTE("Profile data not written to file: %s.\n", "already written"); return 0; } @@ -649,6 +718,62 @@ int __llvm_profile_dump(void) { return rc; } +/* Order file data will be saved in a file with suffx .order. */ +static const char *OrderFileSuffix = ".order"; + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_write_file(void) { + int rc, Length, LengthBeforeAppend, SuffixLength; + const char *Filename; + char *FilenameBuf; + int PDeathSig = 0; + + SuffixLength = strlen(OrderFileSuffix); + Length = getCurFilenameLength() + SuffixLength; + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 1); + + /* Check the filename. */ + if (!Filename) { + PROF_ERR("Failed to write file : %s\n", "Filename not set"); + return -1; + } + + /* Append order file suffix */ + LengthBeforeAppend = strlen(Filename); + memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength); + FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0'; + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + PROF_ERR("Runtime and instrumentation version mismatch : " + "expected %d, but get %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + return -1; + } + + // Temporarily suspend getting SIGKILL when the parent exits. + PDeathSig = lprofSuspendSigKill(); + + /* Write order data to the file. */ + rc = writeOrderFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + + // Restore SIGKILL. + if (PDeathSig == 1) + lprofRestoreSigKill(); + + return rc; +} + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_dump(void) { + int rc = __llvm_orderfile_write_file(); + return rc; +} + static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); } COMPILER_RT_VISIBILITY diff --git a/lib/profile/InstrProfilingInternal.h b/lib/profile/InstrProfilingInternal.h index 40540ab5f205..66f8a0677014 100644 --- a/lib/profile/InstrProfilingInternal.h +++ b/lib/profile/InstrProfilingInternal.h @@ -1,9 +1,8 @@ /*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingMerge.c b/lib/profile/InstrProfilingMerge.c index 2397250fb122..44dce7cc9f62 100644 --- a/lib/profile/InstrProfilingMerge.c +++ b/lib/profile/InstrProfilingMerge.c @@ -1,9 +1,8 @@ /*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------===* |* This file defines the API needed for in-process merging of profile data diff --git a/lib/profile/InstrProfilingMergeFile.c b/lib/profile/InstrProfilingMergeFile.c index dc1bc9762676..b853f15b4737 100644 --- a/lib/profile/InstrProfilingMergeFile.c +++ b/lib/profile/InstrProfilingMergeFile.c @@ -1,9 +1,8 @@ /*===- InstrProfilingMergeFile.c - Profile in-process Merging ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------=== |* This file defines APIs needed to support in-process merging for profile data @@ -21,6 +20,7 @@ /* Merge value profile data pointed to by SrcValueProfData into * in-memory profile counters pointed by to DstData. */ +COMPILER_RT_VISIBILITY void lprofMergeValueProfData(ValueProfData *SrcValueProfData, __llvm_profile_data *DstData) { unsigned I, S, V, DstIndex = 0; diff --git a/lib/profile/InstrProfilingNameVar.c b/lib/profile/InstrProfilingNameVar.c index 264568fbc912..2d67a55b985c 100644 --- a/lib/profile/InstrProfilingNameVar.c +++ b/lib/profile/InstrProfilingNameVar.c @@ -1,9 +1,8 @@ /*===- InstrProfilingNameVar.c - profile name variable setup -------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingPlatformDarwin.c b/lib/profile/InstrProfilingPlatformDarwin.c index 8931abaddf7f..23bdb7f37179 100644 --- a/lib/profile/InstrProfilingPlatformDarwin.c +++ b/lib/profile/InstrProfilingPlatformDarwin.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformDarwin.c - Profile data on Darwin ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -13,28 +12,31 @@ /* Use linker magic to find the bounds of the Data section. */ COMPILER_RT_VISIBILITY extern __llvm_profile_data - DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); + DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME); COMPILER_RT_VISIBILITY extern __llvm_profile_data - DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); + DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME); COMPILER_RT_VISIBILITY extern char - NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); + NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY -extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); +extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY extern uint64_t - CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); + CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY extern uint64_t - CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); + CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern uint32_t + OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); COMPILER_RT_VISIBILITY extern ValueProfNode - VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME_STR); + VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY extern ValueProfNode - VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME_STR); + VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_begin_data(void) { @@ -50,6 +52,8 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; } COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformFuchsia.c b/lib/profile/InstrProfilingPlatformFuchsia.c index a50602dede7d..80beb4145460 100644 --- a/lib/profile/InstrProfilingPlatformFuchsia.c +++ b/lib/profile/InstrProfilingPlatformFuchsia.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformFuchsia.c - Profile data Fuchsia platform ----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -71,7 +70,7 @@ static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, return -1; /* Create VMO to hold the profile data. */ - Status = _zx_vmo_create(0, 0, &__llvm_profile_vmo); + Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &__llvm_profile_vmo); if (Status != ZX_OK) return -1; diff --git a/lib/profile/InstrProfilingPlatformLinux.c b/lib/profile/InstrProfilingPlatformLinux.c index 3764df1d8079..becfe1fd9f5a 100644 --- a/lib/profile/InstrProfilingPlatformLinux.c +++ b/lib/profile/InstrProfilingPlatformLinux.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -14,14 +13,15 @@ #include "InstrProfiling.h" -#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) -#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME) -#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME) -#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_SECT_NAME) -#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_SECT_NAME) -#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_SECT_NAME) -#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_SECT_NAME) -#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_SECT_NAME) +#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) +#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) +#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) +#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) +#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) +#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) +#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) /* Declare section start and stop symbols for various sections * generated by compiler instrumentation. @@ -30,6 +30,7 @@ extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY; extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY; extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY; extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY; +extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY; extern char PROF_NAME_START COMPILER_RT_VISIBILITY; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY; extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY; @@ -37,11 +38,13 @@ extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY; /* Add dummy data to ensure the section is always created. */ __llvm_profile_data - __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME_STR); + __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME); uint64_t - __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME_STR); -char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME_STR); -ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME_STR); + __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME); +uint32_t + __prof_orderfile_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_ORDERFILE_SECT_NAME); +char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME); +ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY const __llvm_profile_data * __llvm_profile_begin_data(void) { @@ -63,6 +66,9 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { + return &PROF_ORDERFILE_START; +} COMPILER_RT_VISIBILITY ValueProfNode * __llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformOther.c b/lib/profile/InstrProfilingPlatformOther.c index 7c2f14cfce16..56c5d8378c29 100644 --- a/lib/profile/InstrProfilingPlatformOther.c +++ b/lib/profile/InstrProfilingPlatformOther.c @@ -1,16 +1,17 @@ /*===- InstrProfilingPlatformOther.c - Profile data default platform ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ -#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ - !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ + !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) && \ + !defined(_WIN32) #include <stdlib.h> +#include <stdio.h> #include "InstrProfiling.h" @@ -20,6 +21,7 @@ static const char *NamesFirst = NULL; static const char *NamesLast = NULL; static uint64_t *CountersFirst = NULL; static uint64_t *CountersLast = NULL; +static uint32_t *OrderFileFirst = NULL; static const void *getMinAddr(const void *A1, const void *A2) { return A1 < A2 ? A1 : A2; @@ -81,6 +83,9 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return CountersLast; } +/* TODO: correctly set up OrderFileFirst. */ +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformWindows.c b/lib/profile/InstrProfilingPlatformWindows.c new file mode 100644 index 000000000000..81b708bb2a38 --- /dev/null +++ b/lib/profile/InstrProfilingPlatformWindows.c @@ -0,0 +1,68 @@ +/*===- InstrProfilingPlatformWindows.c - Profile data on Windows ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +#if defined(_WIN32) + +#if defined(_MSC_VER) +/* Merge read-write sections into .data. */ +#pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfd=.data") +#pragma comment(linker, "/MERGE:.lprfv=.data") +#pragma comment(linker, "/MERGE:.lprfnd=.data") +/* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find + * after the fact. + */ + +/* Allocate read-only section bounds. */ +#pragma section(".lprfn$A", read) +#pragma section(".lprfn$Z", read) + +/* Allocate read-write section bounds. */ +#pragma section(".lprfd$A", read, write) +#pragma section(".lprfd$Z", read, write) +#pragma section(".lprfc$A", read, write) +#pragma section(".lprfc$Z", read, write) +#pragma section(".lorderfile$A", read, write) +#pragma section(".lprfnd$A", read, write) +#pragma section(".lprfnd$Z", read, write) +#endif + +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$A") DataStart = {0}; +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$Z") DataEnd = {0}; + +const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0'; +const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; + +uint64_t COMPILER_RT_SECTION(".lprfc$A") CountersStart; +uint64_t COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; + +ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; +ValueProfNode COMPILER_RT_SECTION(".lprfnd$Z") VNodesEnd; + +const __llvm_profile_data *__llvm_profile_begin_data(void) { + return &DataStart + 1; +} +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } + +const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } +const char *__llvm_profile_end_names(void) { return &NamesEnd; } + +uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } +uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } + +ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } +ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } + +ValueProfNode *CurrentVNode = &VNodesStart + 1; +ValueProfNode *EndVNode = &VNodesEnd; + +#endif diff --git a/lib/profile/InstrProfilingPort.h b/lib/profile/InstrProfilingPort.h index ede6aaaf7886..da5b5c0f8bb7 100644 --- a/lib/profile/InstrProfilingPort.h +++ b/lib/profile/InstrProfilingPort.h @@ -1,9 +1,8 @@ /*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingRuntime.cc b/lib/profile/InstrProfilingRuntime.cc index eb83074983ba..679186ef8309 100644 --- a/lib/profile/InstrProfilingRuntime.cc +++ b/lib/profile/InstrProfilingRuntime.cc @@ -1,9 +1,8 @@ //===- InstrProfilingRuntime.cpp - PGO runtime initialization -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/profile/InstrProfilingUtil.c b/lib/profile/InstrProfilingUtil.c index 083bf14a3260..02d100792db8 100644 --- a/lib/profile/InstrProfilingUtil.c +++ b/lib/profile/InstrProfilingUtil.c @@ -1,9 +1,8 @@ /*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -81,7 +80,7 @@ void *lprofPtrFetchAdd(void **Mem, long ByteIncr) { #endif -#ifdef _MSC_VER +#ifdef _WIN32 COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN]; DWORD BufferSize = sizeof(Buffer); @@ -155,6 +154,26 @@ COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) { #endif } +COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) { + int fd; +#if defined(_WIN32) + fd = _fileno(F); +#else + fd = fileno(F); +#endif + return lprofLockFd(fd); +} + +COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) { + int fd; +#if defined(_WIN32) + fd = _fileno(F); +#else + fd = fileno(F); +#endif + return lprofUnlockFd(fd); +} + COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { FILE *f; int fd; diff --git a/lib/profile/InstrProfilingUtil.h b/lib/profile/InstrProfilingUtil.h index 147677fc8700..efba94ca7639 100644 --- a/lib/profile/InstrProfilingUtil.h +++ b/lib/profile/InstrProfilingUtil.h @@ -1,9 +1,8 @@ /*===- InstrProfilingUtil.h - Support library for PGO instrumentation -----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -24,6 +23,8 @@ unsigned __llvm_profile_get_dir_mode(void); int lprofLockFd(int fd); int lprofUnlockFd(int fd); +int lprofLockFileHandle(FILE *F); +int lprofUnlockFileHandle(FILE *F); /*! Open file \c Filename for read+write with write * lock for exclusive access. The caller will block diff --git a/lib/profile/InstrProfilingValue.c b/lib/profile/InstrProfilingValue.c index c7b01a570649..b7c71768c2c3 100644 --- a/lib/profile/InstrProfilingValue.c +++ b/lib/profile/InstrProfilingValue.c @@ -1,9 +1,8 @@ /*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -32,7 +31,7 @@ static int hasNonDefaultValsPerSite = 0; * allocated by the compiler. */ COMPILER_RT_VISIBILITY ValueProfNode lprofValueProfNodes[INSTR_PROF_VNODE_POOL_SIZE] COMPILER_RT_SECTION( - COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME_STR); + COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); #endif COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite = diff --git a/lib/profile/InstrProfilingWriter.c b/lib/profile/InstrProfilingWriter.c index 7c6061d2c65a..d910cbb8f2fc 100644 --- a/lib/profile/InstrProfilingWriter.c +++ b/lib/profile/InstrProfilingWriter.c @@ -1,9 +1,8 @@ /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/WindowsMMap.h b/lib/profile/WindowsMMap.h index 51a130b31977..c8d6250f41c1 100644 --- a/lib/profile/WindowsMMap.h +++ b/lib/profile/WindowsMMap.h @@ -1,9 +1,8 @@ /*===- WindowsMMap.h - Support library for PGO instrumentation ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/safestack/safestack.cc b/lib/safestack/safestack.cc index e68208015fb9..f713d5e68718 100644 --- a/lib/safestack/safestack.cc +++ b/lib/safestack/safestack.cc @@ -1,9 +1,8 @@ //===-- safestack.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,21 +13,31 @@ // //===----------------------------------------------------------------------===// +#include "safestack_platform.h" +#include "safestack_util.h" + #include <errno.h> -#include <limits.h> -#include <pthread.h> -#include <stddef.h> -#include <stdint.h> -#include <unistd.h> -#include <stdlib.h> #include <sys/resource.h> -#include <sys/types.h> -#if !defined(__NetBSD__) -#include <sys/user.h> -#endif #include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" + +using namespace safestack; + +// TODO: To make accessing the unsafe stack pointer faster, we plan to +// eventually store it directly in the thread control block data structure on +// platforms where this structure is pointed to by %fs or %gs. This is exactly +// the same mechanism as currently being used by the traditional stack +// protector pass to store the stack guard (see getStackCookieLocation() +// function above). Doing so requires changing the tcbhead_t struct in glibc +// on Linux and tcb struct in libc on FreeBSD. +// +// For now, store it in a thread-local variable. +extern "C" { +__attribute__((visibility( + "default"))) __thread void *__safestack_unsafe_stack_ptr = nullptr; +} + +namespace { // TODO: The runtime library does not currently protect the safe stack beyond // relying on the system-enforced ASLR. The protection of the (safe) stack can @@ -73,43 +82,26 @@ const unsigned kStackAlign = 16; /// size rlimit is set to infinity. const unsigned kDefaultUnsafeStackSize = 0x2800000; -/// Runtime page size obtained through sysconf -static unsigned pageSize; - -// TODO: To make accessing the unsafe stack pointer faster, we plan to -// eventually store it directly in the thread control block data structure on -// platforms where this structure is pointed to by %fs or %gs. This is exactly -// the same mechanism as currently being used by the traditional stack -// protector pass to store the stack guard (see getStackCookieLocation() -// function above). Doing so requires changing the tcbhead_t struct in glibc -// on Linux and tcb struct in libc on FreeBSD. -// -// For now, store it in a thread-local variable. -extern "C" { -__attribute__((visibility( - "default"))) __thread void *__safestack_unsafe_stack_ptr = nullptr; -} - // Per-thread unsafe stack information. It's not frequently accessed, so there // it can be kept out of the tcb in normal thread-local variables. -static __thread void *unsafe_stack_start = nullptr; -static __thread size_t unsafe_stack_size = 0; -static __thread size_t unsafe_stack_guard = 0; - -using namespace __sanitizer; - -static inline void *unsafe_stack_alloc(size_t size, size_t guard) { - CHECK_GE(size + guard, size); - void *addr = MmapOrDie(size + guard, "unsafe_stack_alloc"); - MprotectNoAccess((uptr)addr, (uptr)guard); +__thread void *unsafe_stack_start = nullptr; +__thread size_t unsafe_stack_size = 0; +__thread size_t unsafe_stack_guard = 0; + +inline void *unsafe_stack_alloc(size_t size, size_t guard) { + SFS_CHECK(size + guard >= size); + void *addr = Mmap(nullptr, size + guard, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + SFS_CHECK(MAP_FAILED != addr); + Mprotect(addr, guard, PROT_NONE); return (char *)addr + guard; } -static inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { - CHECK_GE((char *)start + size, (char *)start); - CHECK_GE((char *)start + guard, (char *)start); +inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { + SFS_CHECK((char *)start + size >= (char *)start); + SFS_CHECK((char *)start + guard >= (char *)start); void *stack_ptr = (char *)start + size; - CHECK_EQ((((size_t)stack_ptr) & (kStackAlign - 1)), 0); + SFS_CHECK((((size_t)stack_ptr) & (kStackAlign - 1)) == 0); __safestack_unsafe_stack_ptr = stack_ptr; unsafe_stack_start = start; @@ -118,7 +110,7 @@ static inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { } /// Thread data for the cleanup handler -static pthread_key_t thread_cleanup_key; +pthread_key_t thread_cleanup_key; /// Safe stack per-thread information passed to the thread_start function struct tinfo { @@ -132,7 +124,7 @@ struct tinfo { /// Wrap the thread function in order to deallocate the unsafe stack when the /// thread terminates by returning from its main function. -static void *thread_start(void *arg) { +void *thread_start(void *arg) { struct tinfo *tinfo = (struct tinfo *)arg; void *(*start_routine)(void *) = tinfo->start_routine; @@ -154,19 +146,19 @@ struct thread_stack_ll { void *stack_base; size_t size; pid_t pid; - tid_t tid; + ThreadId tid; }; /// Linked list of unsafe stacks for threads that are exiting. We delay /// unmapping them until the thread exits. -static thread_stack_ll *thread_stacks = nullptr; -static pthread_mutex_t thread_stacks_mutex = PTHREAD_MUTEX_INITIALIZER; +thread_stack_ll *thread_stacks = nullptr; +pthread_mutex_t thread_stacks_mutex = PTHREAD_MUTEX_INITIALIZER; /// Thread-specific data destructor. We want to free the unsafe stack only after /// this thread is terminated. libc can call functions in safestack-instrumented /// code (like free) after thread-specific data destructors have run. -static void thread_cleanup_handler(void *_iter) { - CHECK_NE(unsafe_stack_start, nullptr); +void thread_cleanup_handler(void *_iter) { + SFS_CHECK(unsafe_stack_start != nullptr); pthread_setspecific(thread_cleanup_key, NULL); pthread_mutex_lock(&thread_stacks_mutex); @@ -177,17 +169,15 @@ static void thread_cleanup_handler(void *_iter) { pthread_mutex_unlock(&thread_stacks_mutex); pid_t pid = getpid(); - tid_t tid = GetTid(); + ThreadId tid = GetTid(); // Free stacks for dead threads thread_stack_ll **stackp = &temp_stacks; while (*stackp) { thread_stack_ll *stack = *stackp; - int error; if (stack->pid != pid || - (internal_iserror(TgKill(stack->pid, stack->tid, 0), &error) && - error == ESRCH)) { - UnmapOrDie(stack->stack_base, stack->size); + (-1 == TgKill(stack->pid, stack->tid, 0) && errno == ESRCH)) { + Munmap(stack->stack_base, stack->size); *stackp = stack->next; free(stack); } else @@ -212,7 +202,7 @@ static void thread_cleanup_handler(void *_iter) { unsafe_stack_start = nullptr; } -static void EnsureInterceptorsInitialized(); +void EnsureInterceptorsInitialized(); /// Intercept thread creation operation to allocate and setup the unsafe stack INTERCEPTOR(int, pthread_create, pthread_t *thread, @@ -234,11 +224,12 @@ INTERCEPTOR(int, pthread_create, pthread_t *thread, pthread_attr_destroy(&tmpattr); } - CHECK_NE(size, 0); - CHECK_EQ((size & (kStackAlign - 1)), 0); - CHECK_EQ((guard & (pageSize - 1)), 0); + SFS_CHECK(size); + size = RoundUpTo(size, kStackAlign); void *addr = unsafe_stack_alloc(size, guard); + // Put tinfo at the end of the buffer. guard may be not page aligned. + // If that is so then some bytes after addr can be mprotected. struct tinfo *tinfo = (struct tinfo *)(((char *)addr) + size - sizeof(struct tinfo)); tinfo->start_routine = start_routine; @@ -250,12 +241,13 @@ INTERCEPTOR(int, pthread_create, pthread_t *thread, return REAL(pthread_create)(thread, attr, thread_start, tinfo); } -static BlockingMutex interceptor_init_lock(LINKER_INITIALIZED); -static bool interceptors_inited = false; +pthread_mutex_t interceptor_init_mutex = PTHREAD_MUTEX_INITIALIZER; +bool interceptors_inited = false; -static void EnsureInterceptorsInitialized() { - BlockingMutexLock lock(&interceptor_init_lock); - if (interceptors_inited) return; +void EnsureInterceptorsInitialized() { + MutexLock lock(interceptor_init_mutex); + if (interceptors_inited) + return; // Initialize pthread interceptors for thread allocation INTERCEPT_FUNCTION(pthread_create); @@ -263,6 +255,8 @@ static void EnsureInterceptorsInitialized() { interceptors_inited = true; } +} // namespace + extern "C" __attribute__((visibility("default"))) #if !SANITIZER_CAN_USE_PREINIT_ARRAY // On ELF platforms, the constructor is invoked using .preinit_array (see below) @@ -279,9 +273,7 @@ void __safestack_init() { // Allocate unsafe stack for main thread void *addr = unsafe_stack_alloc(size, guard); - unsafe_stack_setup(addr, size, guard); - pageSize = sysconf(_SC_PAGESIZE); // Setup the cleanup handler pthread_key_create(&thread_cleanup_key, thread_cleanup_handler); diff --git a/lib/safestack/safestack_platform.h b/lib/safestack/safestack_platform.h new file mode 100644 index 000000000000..81e4c2645ce2 --- /dev/null +++ b/lib/safestack/safestack_platform.h @@ -0,0 +1,124 @@ +//===-- safestack_platform.h ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements platform specific parts of SafeStack runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef SAFESTACK_PLATFORM_H +#define SAFESTACK_PLATFORM_H + +#include "safestack_util.h" +#include "sanitizer_common/sanitizer_platform.h" + +#include <dlfcn.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <unistd.h> + +#if !(SANITIZER_NETBSD || SANITIZER_FREEBSD || SANITIZER_LINUX) +#error "Support for your platform has not been implemented" +#endif + +#if SANITIZER_NETBSD +#include <lwp.h> + +extern "C" void *__mmap(void *, size_t, int, int, int, int, off_t); +#endif + +#if SANITIZER_FREEBSD +#include <sys/thr.h> +#endif + +namespace safestack { + +#if SANITIZER_NETBSD +static void *GetRealLibcAddress(const char *symbol) { + void *real = dlsym(RTLD_NEXT, symbol); + if (!real) + real = dlsym(RTLD_DEFAULT, symbol); + if (!real) { + fprintf(stderr, "safestack GetRealLibcAddress failed for symbol=%s", + symbol); + abort(); + } + return real; +} + +#define _REAL(func, ...) real##_##func(__VA_ARGS__) +#define DEFINE__REAL(ret_type, func, ...) \ + static ret_type (*real_##func)(__VA_ARGS__) = NULL; \ + if (!real_##func) { \ + real_##func = (ret_type(*)(__VA_ARGS__))GetRealLibcAddress(#func); \ + } \ + SFS_CHECK(real_##func); +#endif + +using ThreadId = uint64_t; + +inline ThreadId GetTid() { +#if SANITIZER_NETBSD + DEFINE__REAL(int, _lwp_self); + return _REAL(_lwp_self); +#elif SANITIZER_FREEBSD + long Tid; + thr_self(&Tid); + return Tid; +#else + return syscall(SYS_gettid); +#endif +} + +inline int TgKill(pid_t pid, ThreadId tid, int sig) { +#if SANITIZER_NETBSD + DEFINE__REAL(int, _lwp_kill, int a, int b); + (void)pid; + return _REAL(_lwp_kill, tid, sig); +#elif SANITIZER_FREEBSD + return syscall(SYS_thr_kill2, pid, tid, sig); +#else + return syscall(SYS_tgkill, pid, tid, sig); +#endif +} + +inline void *Mmap(void *addr, size_t length, int prot, int flags, int fd, + off_t offset) { +#if SANITIZER_NETBSD + return __mmap(addr, length, prot, flags, fd, 0, offset); +#elif defined(__x86_64__) && (SANITIZER_FREEBSD) + return (void *)__syscall(SYS_mmap, addr, length, prot, flags, fd, offset); +#else + return (void *)syscall(SYS_mmap, addr, length, prot, flags, fd, offset); +#endif +} + +inline int Munmap(void *addr, size_t length) { +#if SANITIZER_NETBSD + DEFINE__REAL(int, munmap, void *a, size_t b); + return _REAL(munmap, addr, length); +#else + return syscall(SYS_munmap, addr, length); +#endif +} + +inline int Mprotect(void *addr, size_t length, int prot) { +#if SANITIZER_NETBSD + DEFINE__REAL(int, mprotect, void *a, size_t b, int c); + return _REAL(mprotect, addr, length, prot); +#else + return syscall(SYS_mprotect, addr, length, prot); +#endif +} + +} // namespace safestack + +#endif // SAFESTACK_PLATFORM_H diff --git a/lib/safestack/safestack_util.h b/lib/safestack/safestack_util.h new file mode 100644 index 000000000000..da3f11b54cab --- /dev/null +++ b/lib/safestack/safestack_util.h @@ -0,0 +1,49 @@ +//===-- safestack_util.h --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains utility code for SafeStack implementation. +// +//===----------------------------------------------------------------------===// + +#ifndef SAFESTACK_UTIL_H +#define SAFESTACK_UTIL_H + +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> + +namespace safestack { + +#define SFS_CHECK(a) \ + do { \ + if (!(a)) { \ + fprintf(stderr, "safestack CHECK failed: %s:%d %s\n", __FILE__, \ + __LINE__, #a); \ + abort(); \ + }; \ + } while (false) + +inline size_t RoundUpTo(size_t size, size_t boundary) { + SFS_CHECK((boundary & (boundary - 1)) == 0); + return (size + boundary - 1) & ~(boundary - 1); +} + +class MutexLock { + public: + explicit MutexLock(pthread_mutex_t &mutex) : mutex_(&mutex) { + pthread_mutex_lock(mutex_); + } + ~MutexLock() { pthread_mutex_unlock(mutex_); } + + private: + pthread_mutex_t *mutex_ = nullptr; +}; + +} // namespace safestack + +#endif // SAFESTACK_UTIL_H diff --git a/lib/sanitizer_common/sancov_begin.S b/lib/sanitizer_common/sancov_begin.S deleted file mode 100644 index c8ad0a0bcb5c..000000000000 --- a/lib/sanitizer_common/sancov_begin.S +++ /dev/null @@ -1,5 +0,0 @@ - .type __start___sancov_guards,@object - .globl __start___sancov_guards - .section __sancov_guards,"aw",@progbits - .p2align 2 -__start___sancov_guards: diff --git a/lib/sanitizer_common/sancov_end.S b/lib/sanitizer_common/sancov_end.S deleted file mode 100644 index 31117b1c0b56..000000000000 --- a/lib/sanitizer_common/sancov_end.S +++ /dev/null @@ -1,5 +0,0 @@ - .type __stop___sancov_guards,@object - .globl __stop___sancov_guards - .section __sancov_guards,"aw",@progbits - .p2align 2 -__stop___sancov_guards: diff --git a/lib/sanitizer_common/sancov_flags.cc b/lib/sanitizer_common/sancov_flags.cc index 9abb5b5c5988..ec6c14b1e2e4 100644 --- a/lib/sanitizer_common/sancov_flags.cc +++ b/lib/sanitizer_common/sancov_flags.cc @@ -1,9 +1,8 @@ //===-- sancov_flags.cc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -50,7 +49,7 @@ void InitializeSancovFlags() { RegisterSancovFlags(&parser, f); parser.ParseString(MaybeCallSancovDefaultOptions()); - parser.ParseString(GetEnv("SANCOV_OPTIONS")); + parser.ParseStringFromEnv("SANCOV_OPTIONS"); ReportUnrecognizedFlags(); if (f->help) parser.PrintFlagDescriptions(); diff --git a/lib/sanitizer_common/sancov_flags.h b/lib/sanitizer_common/sancov_flags.h index 627d9a3df6f4..95d4ee5ca43d 100644 --- a/lib/sanitizer_common/sancov_flags.h +++ b/lib/sanitizer_common/sancov_flags.h @@ -1,9 +1,8 @@ //===-- sancov_flags.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sancov_flags.inc b/lib/sanitizer_common/sancov_flags.inc index 63a1f0cbc49d..cca33fc359f4 100644 --- a/lib/sanitizer_common/sancov_flags.inc +++ b/lib/sanitizer_common/sancov_flags.inc @@ -1,9 +1,8 @@ //===-- sancov_flags.inc ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_addrhashmap.h b/lib/sanitizer_common/sanitizer_addrhashmap.h index 2ca3c405bff3..a033e788cbf8 100644 --- a/lib/sanitizer_common/sanitizer_addrhashmap.h +++ b/lib/sanitizer_common/sanitizer_addrhashmap.h @@ -1,9 +1,8 @@ //===-- sanitizer_addrhashmap.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator.cc b/lib/sanitizer_common/sanitizer_allocator.cc index 6bfd5e5eea50..1739bb66b657 100644 --- a/lib/sanitizer_common/sanitizer_allocator.cc +++ b/lib/sanitizer_common/sanitizer_allocator.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -171,6 +170,18 @@ void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { return (char*)p + sizeof(u64); } +void *InternalReallocArray(void *addr, uptr count, uptr size, + InternalAllocatorCache *cache) { + if (UNLIKELY(CheckForCallocOverflow(count, size))) { + Report( + "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) " + "cannot be represented in type size_t\n", + SanitizerToolName, count, size); + Die(); + } + return InternalRealloc(addr, count * size, cache); +} + void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { if (UNLIKELY(CheckForCallocOverflow(count, size))) { Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) " diff --git a/lib/sanitizer_common/sanitizer_allocator.h b/lib/sanitizer_common/sanitizer_allocator.h index 88017160acf9..23d589888d3b 100644 --- a/lib/sanitizer_common/sanitizer_allocator.h +++ b/lib/sanitizer_common/sanitizer_allocator.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_bytemap.h b/lib/sanitizer_common/sanitizer_allocator_bytemap.h index ef26941fe609..0084bb62c83c 100644 --- a/lib/sanitizer_common/sanitizer_allocator_bytemap.h +++ b/lib/sanitizer_common/sanitizer_allocator_bytemap.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_bytemap.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_checks.cc b/lib/sanitizer_common/sanitizer_allocator_checks.cc index dc263dbef5fc..bb56010f1074 100644 --- a/lib/sanitizer_common/sanitizer_allocator_checks.cc +++ b/lib/sanitizer_common/sanitizer_allocator_checks.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_checks.cc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_checks.h b/lib/sanitizer_common/sanitizer_allocator_checks.h index 61bd26e68cc6..f436ce9ecdea 100644 --- a/lib/sanitizer_common/sanitizer_allocator_checks.h +++ b/lib/sanitizer_common/sanitizer_allocator_checks.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_checks.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_combined.h b/lib/sanitizer_common/sanitizer_allocator_combined.h index fcc4469c98cf..33f89d6d4992 100644 --- a/lib/sanitizer_common/sanitizer_allocator_combined.h +++ b/lib/sanitizer_common/sanitizer_allocator_combined.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_combined.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,18 +19,15 @@ // When allocating 2^x bytes it should return 2^x aligned chunk. // PrimaryAllocator is used via a local AllocatorCache. // SecondaryAllocator can allocate anything, but is not efficient. -template <class PrimaryAllocator, class AllocatorCache, - class SecondaryAllocator, - typename AddressSpaceViewTy = LocalAddressSpaceView> // NOLINT +template <class PrimaryAllocator, + class LargeMmapAllocatorPtrArray = DefaultLargeMmapAllocatorPtrArray> class CombinedAllocator { public: - using AddressSpaceView = AddressSpaceViewTy; - static_assert(is_same<AddressSpaceView, - typename PrimaryAllocator::AddressSpaceView>::value, - "PrimaryAllocator is using wrong AddressSpaceView"); - static_assert(is_same<AddressSpaceView, - typename SecondaryAllocator::AddressSpaceView>::value, - "SecondaryAllocator is using wrong AddressSpaceView"); + using AllocatorCache = typename PrimaryAllocator::AllocatorCache; + using SecondaryAllocator = + LargeMmapAllocator<typename PrimaryAllocator::MapUnmapCallback, + LargeMmapAllocatorPtrArray, + typename PrimaryAllocator::AddressSpaceView>; void InitLinkerInitialized(s32 release_to_os_interval_ms) { stats_.InitLinkerInitialized(); diff --git a/lib/sanitizer_common/sanitizer_allocator_interface.h b/lib/sanitizer_common/sanitizer_allocator_interface.h index 2f5ce3151ee4..c1b27563e2fc 100644 --- a/lib/sanitizer_common/sanitizer_allocator_interface.h +++ b/lib/sanitizer_common/sanitizer_allocator_interface.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_interface.h ------------------------- C++ -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_internal.h b/lib/sanitizer_common/sanitizer_allocator_internal.h index 30fc7042b6b5..32849036fd04 100644 --- a/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_internal.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,42 +22,30 @@ namespace __sanitizer { // purposes. typedef CompactSizeClassMap InternalSizeClassMap; -static const uptr kInternalAllocatorRegionSizeLog = 20; -static const uptr kInternalAllocatorNumRegions = - SANITIZER_MMAP_RANGE_SIZE >> kInternalAllocatorRegionSizeLog; -#if SANITIZER_WORDSIZE == 32 -typedef FlatByteMap<kInternalAllocatorNumRegions> ByteMap; -#else -typedef TwoLevelByteMap<(kInternalAllocatorNumRegions >> 12), 1 << 12> ByteMap; -#endif struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef InternalSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = kInternalAllocatorRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __sanitizer::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; typedef SizeClassAllocator32<AP32> PrimaryInternalAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryInternalAllocator> - InternalAllocatorCache; - -typedef LargeMmapAllocator<NoOpMapUnmapCallback, - LargeMmapAllocatorPtrArrayStatic> - SecondaryInternalAllocator; - -typedef CombinedAllocator<PrimaryInternalAllocator, InternalAllocatorCache, - SecondaryInternalAllocator> InternalAllocator; +typedef CombinedAllocator<PrimaryInternalAllocator, + LargeMmapAllocatorPtrArrayStatic> + InternalAllocator; +typedef InternalAllocator::AllocatorCache InternalAllocatorCache; void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr, uptr alignment = 0); void *InternalRealloc(void *p, uptr size, InternalAllocatorCache *cache = nullptr); -void *InternalCalloc(uptr countr, uptr size, +void *InternalReallocArray(void *p, uptr count, uptr size, + InternalAllocatorCache *cache = nullptr); +void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache = nullptr); void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); InternalAllocator *internal_allocator(); diff --git a/lib/sanitizer_common/sanitizer_allocator_local_cache.h b/lib/sanitizer_common/sanitizer_allocator_local_cache.h index 1bb8fc27defa..108dfc231a22 100644 --- a/lib/sanitizer_common/sanitizer_allocator_local_cache.h +++ b/lib/sanitizer_common/sanitizer_allocator_local_cache.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_local_cache.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,13 +13,6 @@ #error This file must be included inside sanitizer_allocator.h #endif -// Objects of this type should be used as local caches for SizeClassAllocator64 -// or SizeClassAllocator32. Since the typical use of this class is to have one -// object per thread in TLS, is has to be POD. -template<class SizeClassAllocator> -struct SizeClassAllocatorLocalCache - : SizeClassAllocator::AllocatorCache {}; - // Cache used by SizeClassAllocator64. template <class SizeClassAllocator> struct SizeClassAllocator64LocalCache { diff --git a/lib/sanitizer_common/sanitizer_allocator_primary32.h b/lib/sanitizer_common/sanitizer_allocator_primary32.h index abaac3d1a70b..3b1838b3985a 100644 --- a/lib/sanitizer_common/sanitizer_allocator_primary32.h +++ b/lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_primary32.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -47,6 +46,11 @@ struct SizeClassAllocator32FlagMasks { // Bit masks. template <class Params> class SizeClassAllocator32 { + private: + static const u64 kTwoLevelByteMapSize1 = + (Params::kSpaceSize >> Params::kRegionSizeLog) >> 12; + static const u64 kMinFirstMapSizeTwoLevelByteMap = 4; + public: using AddressSpaceView = typename Params::AddressSpaceView; static const uptr kSpaceBeg = Params::kSpaceBeg; @@ -54,12 +58,15 @@ class SizeClassAllocator32 { static const uptr kMetadataSize = Params::kMetadataSize; typedef typename Params::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = Params::kRegionSizeLog; - typedef typename Params::ByteMap ByteMap; typedef typename Params::MapUnmapCallback MapUnmapCallback; + using ByteMap = typename conditional< + (kTwoLevelByteMapSize1 < kMinFirstMapSizeTwoLevelByteMap), + FlatByteMap<(Params::kSpaceSize >> Params::kRegionSizeLog), + AddressSpaceView>, + TwoLevelByteMap<kTwoLevelByteMapSize1, 1 << 12, AddressSpaceView>>::type; - static_assert( - is_same<typename ByteMap::AddressSpaceView, AddressSpaceView>::value, - "AddressSpaceView type mismatch"); + COMPILER_CHECK(!SANITIZER_SIGN_EXTENDED_ADDRESSES || + (kSpaceSize & (kSpaceSize - 1)) == 0); static const bool kRandomShuffleChunks = Params::kFlags & SizeClassAllocator32FlagMasks::kRandomShuffleChunks; @@ -182,6 +189,8 @@ class SizeClassAllocator32 { bool PointerIsMine(const void *p) { uptr mem = reinterpret_cast<uptr>(p); + if (SANITIZER_SIGN_EXTENDED_ADDRESSES) + mem &= (kSpaceSize - 1); if (mem < kSpaceBeg || mem >= kSpaceBeg + kSpaceSize) return false; return GetSizeClass(p) != 0; @@ -207,7 +216,7 @@ class SizeClassAllocator32 { return ClassIdToSize(GetSizeClass(p)); } - uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } + static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } uptr TotalMemoryUsed() { // No need to lock here. @@ -273,7 +282,9 @@ class SizeClassAllocator32 { }; COMPILER_CHECK(sizeof(SizeClassInfo) % kCacheLineSize == 0); - uptr ComputeRegionId(uptr mem) { + uptr ComputeRegionId(uptr mem) const { + if (SANITIZER_SIGN_EXTENDED_ADDRESSES) + mem &= (kSpaceSize - 1); const uptr res = mem >> kRegionSizeLog; CHECK_LT(res, kNumPossibleRegions); return res; diff --git a/lib/sanitizer_common/sanitizer_allocator_primary64.h b/lib/sanitizer_common/sanitizer_allocator_primary64.h index b063bf0d302f..90603280e7c9 100644 --- a/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ b/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_primary64.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -81,7 +80,8 @@ class SizeClassAllocator64 { CHECK_NE(NonConstSpaceBeg, ~(uptr)0); } SetReleaseToOSIntervalMs(release_to_os_interval_ms); - MapWithCallbackOrDie(SpaceEnd(), AdditionalSize()); + MapWithCallbackOrDie(SpaceEnd(), AdditionalSize(), + "SizeClassAllocator: region info"); // Check that the RegionInfo array is aligned on the CacheLine size. DCHECK_EQ(SpaceEnd() % kCacheLineSize, 0); } @@ -154,7 +154,7 @@ class SizeClassAllocator64 { return true; } - bool PointerIsMine(const void *p) { + bool PointerIsMine(const void *p) const { uptr P = reinterpret_cast<uptr>(p); if (kUsingConstantSpaceBeg && (kSpaceBeg % kSpaceSize) == 0) return P / kSpaceSize == kSpaceBeg / kSpaceSize; @@ -189,7 +189,7 @@ class SizeClassAllocator64 { uptr beg = chunk_idx * size; uptr next_beg = beg + size; if (class_id >= kNumClasses) return nullptr; - RegionInfo *region = GetRegionInfo(class_id); + const RegionInfo *region = AddressSpaceView::Load(GetRegionInfo(class_id)); if (region->mapped_user >= next_beg) return reinterpret_cast<void*>(reg_beg + beg); return nullptr; @@ -200,7 +200,7 @@ class SizeClassAllocator64 { return ClassIdToSize(GetSizeClass(p)); } - uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } + static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } void *GetMetaData(const void *p) { uptr class_id = GetSizeClass(p); @@ -634,8 +634,8 @@ class SizeClassAllocator64 { return reinterpret_cast<CompactPtrT *>(GetMetadataEnd(region_beg)); } - bool MapWithCallback(uptr beg, uptr size) { - uptr mapped = address_range.Map(beg, size); + bool MapWithCallback(uptr beg, uptr size, const char *name) { + uptr mapped = address_range.Map(beg, size, name); if (UNLIKELY(!mapped)) return false; CHECK_EQ(beg, mapped); @@ -643,8 +643,8 @@ class SizeClassAllocator64 { return true; } - void MapWithCallbackOrDie(uptr beg, uptr size) { - CHECK_EQ(beg, address_range.MapOrDie(beg, size)); + void MapWithCallbackOrDie(uptr beg, uptr size, const char *name) { + CHECK_EQ(beg, address_range.MapOrDie(beg, size, name)); MapUnmapCallback().OnMap(beg, size); } @@ -662,7 +662,8 @@ class SizeClassAllocator64 { uptr current_map_end = reinterpret_cast<uptr>(GetFreeArray(region_beg)) + region->mapped_free_array; uptr new_map_size = new_mapped_free_array - region->mapped_free_array; - if (UNLIKELY(!MapWithCallback(current_map_end, new_map_size))) + if (UNLIKELY(!MapWithCallback(current_map_end, new_map_size, + "SizeClassAllocator: freearray"))) return false; region->mapped_free_array = new_mapped_free_array; } @@ -713,7 +714,8 @@ class SizeClassAllocator64 { if (UNLIKELY(IsRegionExhausted(region, class_id, user_map_size))) return false; if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user, - user_map_size))) + user_map_size, + "SizeClassAllocator: region data"))) return false; stat->Add(AllocatorStatMapped, user_map_size); region->mapped_user += user_map_size; @@ -733,7 +735,7 @@ class SizeClassAllocator64 { return false; if (UNLIKELY(!MapWithCallback( GetMetadataEnd(region_beg) - region->mapped_meta - meta_map_size, - meta_map_size))) + meta_map_size, "SizeClassAllocator: region metadata"))) return false; region->mapped_meta += meta_map_size; } diff --git a/lib/sanitizer_common/sanitizer_allocator_report.cc b/lib/sanitizer_common/sanitizer_allocator_report.cc index e93f90c2a4b7..dfc418166556 100644 --- a/lib/sanitizer_common/sanitizer_allocator_report.cc +++ b/lib/sanitizer_common/sanitizer_allocator_report.cc @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_report.cc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -52,6 +51,18 @@ void NORETURN ReportCallocOverflow(uptr count, uptr size, Die(); } +void NORETURN ReportReallocArrayOverflow(uptr count, uptr size, + const StackTrace *stack) { + { + ScopedAllocatorErrorReport report("reallocarray-overflow", stack); + Report( + "ERROR: %s: reallocarray parameters overflow: count * size (%zd * %zd) " + "cannot be represented in type size_t\n", + SanitizerToolName, count, size); + } + Die(); +} + void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack) { { ScopedAllocatorErrorReport report("pvalloc-overflow", stack); diff --git a/lib/sanitizer_common/sanitizer_allocator_report.h b/lib/sanitizer_common/sanitizer_allocator_report.h index b19b22fa5359..0653c365c1cd 100644 --- a/lib/sanitizer_common/sanitizer_allocator_report.h +++ b/lib/sanitizer_common/sanitizer_allocator_report.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_report.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -22,6 +21,8 @@ namespace __sanitizer { void NORETURN ReportCallocOverflow(uptr count, uptr size, const StackTrace *stack); +void NORETURN ReportReallocArrayOverflow(uptr count, uptr size, + const StackTrace *stack); void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack); void NORETURN ReportInvalidAllocationAlignment(uptr alignment, const StackTrace *stack); diff --git a/lib/sanitizer_common/sanitizer_allocator_secondary.h b/lib/sanitizer_common/sanitizer_allocator_secondary.h index 0c8505c34c87..1d128f55de05 100644 --- a/lib/sanitizer_common/sanitizer_allocator_secondary.h +++ b/lib/sanitizer_common/sanitizer_allocator_secondary.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_secondary.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -184,22 +183,25 @@ class LargeMmapAllocator { uptr p = reinterpret_cast<uptr>(ptr); SpinMutexLock l(&mutex_); uptr nearest_chunk = 0; + Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_); // Cache-friendly linear search. for (uptr i = 0; i < n_chunks_; i++) { - uptr ch = reinterpret_cast<uptr>(chunks_[i]); + uptr ch = reinterpret_cast<uptr>(chunks[i]); if (p < ch) continue; // p is at left to this chunk, skip it. if (p - ch < p - nearest_chunk) nearest_chunk = ch; } if (!nearest_chunk) return nullptr; - Header *h = reinterpret_cast<Header *>(nearest_chunk); + const Header *h = + AddressSpaceView::Load(reinterpret_cast<Header *>(nearest_chunk)); + Header *h_ptr = reinterpret_cast<Header *>(nearest_chunk); CHECK_GE(nearest_chunk, h->map_beg); CHECK_LT(nearest_chunk, h->map_beg + h->map_size); CHECK_LE(nearest_chunk, p); if (h->map_beg + h->map_size <= p) return nullptr; - return GetUser(h); + return GetUser(h_ptr); } void EnsureSortedChunks() { @@ -219,9 +221,10 @@ class LargeMmapAllocator { uptr n = n_chunks_; if (!n) return nullptr; EnsureSortedChunks(); - auto min_mmap_ = reinterpret_cast<uptr>(chunks_[0]); - auto max_mmap_ = - reinterpret_cast<uptr>(chunks_[n - 1]) + chunks_[n - 1]->map_size; + Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_); + auto min_mmap_ = reinterpret_cast<uptr>(chunks[0]); + auto max_mmap_ = reinterpret_cast<uptr>(chunks[n - 1]) + + AddressSpaceView::Load(chunks[n - 1])->map_size; if (p < min_mmap_ || p >= max_mmap_) return nullptr; uptr beg = 0, end = n - 1; @@ -229,23 +232,24 @@ class LargeMmapAllocator { // to avoid expensive cache-thrashing loads. while (end - beg >= 2) { uptr mid = (beg + end) / 2; // Invariant: mid >= beg + 1 - if (p < reinterpret_cast<uptr>(chunks_[mid])) - end = mid - 1; // We are not interested in chunks_[mid]. + if (p < reinterpret_cast<uptr>(chunks[mid])) + end = mid - 1; // We are not interested in chunks[mid]. else - beg = mid; // chunks_[mid] may still be what we want. + beg = mid; // chunks[mid] may still be what we want. } if (beg < end) { CHECK_EQ(beg + 1, end); // There are 2 chunks left, choose one. - if (p >= reinterpret_cast<uptr>(chunks_[end])) + if (p >= reinterpret_cast<uptr>(chunks[end])) beg = end; } - Header *h = chunks_[beg]; + const Header *h = AddressSpaceView::Load(chunks[beg]); + Header *h_ptr = chunks[beg]; if (h->map_beg + h->map_size <= p || p < h->map_beg) return nullptr; - return GetUser(h); + return GetUser(h_ptr); } void PrintStats() { diff --git a/lib/sanitizer_common/sanitizer_allocator_size_class_map.h b/lib/sanitizer_common/sanitizer_allocator_size_class_map.h index 1c05fb8ff06b..12d8c8923071 100644 --- a/lib/sanitizer_common/sanitizer_allocator_size_class_map.h +++ b/lib/sanitizer_common/sanitizer_allocator_size_class_map.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_size_class_map.h --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_allocator_stats.h b/lib/sanitizer_common/sanitizer_allocator_stats.h index 76df3ed8de53..6f14e3863c31 100644 --- a/lib/sanitizer_common/sanitizer_allocator_stats.h +++ b/lib/sanitizer_common/sanitizer_allocator_stats.h @@ -1,9 +1,8 @@ //===-- sanitizer_allocator_stats.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_asm.h b/lib/sanitizer_common/sanitizer_asm.h index 76a0bf71425d..184d118d97d8 100644 --- a/lib/sanitizer_common/sanitizer_asm.h +++ b/lib/sanitizer_common/sanitizer_asm.h @@ -1,9 +1,8 @@ //===-- sanitizer_asm.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,14 +44,23 @@ #if !defined(__APPLE__) # define ASM_HIDDEN(symbol) .hidden symbol -# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function +# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function # define ASM_SIZE(symbol) .size symbol, .-symbol # define ASM_SYMBOL(symbol) symbol # define ASM_SYMBOL_INTERCEPTOR(symbol) symbol +# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol #else # define ASM_HIDDEN(symbol) # define ASM_TYPE_FUNCTION(symbol) # define ASM_SIZE(symbol) # define ASM_SYMBOL(symbol) _##symbol # define ASM_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol +# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol +#endif + +#if defined(__ELF__) && (defined(__GNU__) || defined(__FreeBSD__) || \ + defined(__Fuchsia__) || defined(__linux__)) +#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits // NOLINT +#else +#define NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/sanitizer_common/sanitizer_atomic.h b/lib/sanitizer_common/sanitizer_atomic.h index 8f400acc999c..a798a0cf25d9 100644 --- a/lib/sanitizer_common/sanitizer_atomic.h +++ b/lib/sanitizer_common/sanitizer_atomic.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang.h b/lib/sanitizer_common/sanitizer_atomic_clang.h index cd41c920f1b1..c40461ebc3bf 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang_mips.h b/lib/sanitizer_common/sanitizer_atomic_clang_mips.h index c64e6e542798..d369aeb9935c 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang_mips.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang_mips.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang_mips.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang_other.h b/lib/sanitizer_common/sanitizer_atomic_clang_other.h index 2eea549c0fdd..b8685a854267 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang_other.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang_other.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang_other.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_clang_x86.h b/lib/sanitizer_common/sanitizer_atomic_clang_x86.h index 195533ea2320..f2ce553baa7a 100644 --- a/lib/sanitizer_common/sanitizer_atomic_clang_x86.h +++ b/lib/sanitizer_common/sanitizer_atomic_clang_x86.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_clang_x86.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_atomic_msvc.h b/lib/sanitizer_common/sanitizer_atomic_msvc.h index 6d94056d8c68..a249657d6617 100644 --- a/lib/sanitizer_common/sanitizer_atomic_msvc.h +++ b/lib/sanitizer_common/sanitizer_atomic_msvc.h @@ -1,9 +1,8 @@ //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_bitvector.h b/lib/sanitizer_common/sanitizer_bitvector.h index 0f65814c398f..07a59ab11c42 100644 --- a/lib/sanitizer_common/sanitizer_bitvector.h +++ b/lib/sanitizer_common/sanitizer_bitvector.h @@ -1,9 +1,8 @@ //===-- sanitizer_bitvector.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_bvgraph.h b/lib/sanitizer_common/sanitizer_bvgraph.h index 9c2db9ae4954..e7249055bee6 100644 --- a/lib/sanitizer_common/sanitizer_bvgraph.h +++ b/lib/sanitizer_common/sanitizer_bvgraph.h @@ -1,9 +1,8 @@ //===-- sanitizer_bvgraph.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common.cc b/lib/sanitizer_common/sanitizer_common.cc index 6868961902ce..80fb8f60fc0b 100644 --- a/lib/sanitizer_common/sanitizer_common.cc +++ b/lib/sanitizer_common/sanitizer_common.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h index d0aebd994129..1703899e32b5 100644 --- a/lib/sanitizer_common/sanitizer_common.h +++ b/lib/sanitizer_common/sanitizer_common.h @@ -1,9 +1,8 @@ //===-- sanitizer_common.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -102,10 +101,11 @@ void *MmapOrDieOnFatalError(uptr size, const char *mem_type); bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr) WARN_UNUSED_RESULT; void *MmapNoReserveOrDie(uptr size, const char *mem_type); -void *MmapFixedOrDie(uptr fixed_addr, uptr size); +void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name = nullptr); // Behaves just like MmapFixedOrDie, but tolerates out of memory condition, in // that case returns nullptr. -void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size); +void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, + const char *name = nullptr); void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr); void *MmapNoAccess(uptr size); // Map aligned chunk of address space; size and alignment are powers of two. @@ -141,8 +141,8 @@ void RunFreeHooks(const void *ptr); class ReservedAddressRange { public: uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0); - uptr Map(uptr fixed_addr, uptr size); - uptr MapOrDie(uptr fixed_addr, uptr size); + uptr Map(uptr fixed_addr, uptr size, const char *name = nullptr); + uptr MapOrDie(uptr fixed_addr, uptr size, const char *name = nullptr); void Unmap(uptr addr, uptr size); void *base() const { return base_; } uptr size() const { return size_; } @@ -238,7 +238,6 @@ char **GetArgv(); char **GetEnviron(); void PrintCmdline(); bool StackSizeIsUnlimited(); -uptr GetStackSizeLimitInBytes(); void SetStackSizeLimitInBytes(uptr limit); bool AddressSpaceIsUnlimited(); void SetAddressSpaceUnlimited(); @@ -804,7 +803,13 @@ enum AndroidApiLevel { void WriteToSyslog(const char *buffer); -#if SANITIZER_MAC +#if defined(SANITIZER_WINDOWS) && defined(_MSC_VER) && !defined(__clang__) +#define SANITIZER_WIN_TRACE 1 +#else +#define SANITIZER_WIN_TRACE 0 +#endif + +#if SANITIZER_MAC || SANITIZER_WIN_TRACE void LogFullErrorReport(const char *buffer); #else INLINE void LogFullErrorReport(const char *buffer) {} @@ -818,7 +823,7 @@ INLINE void WriteOneLineToSyslog(const char *s) {} INLINE void LogMessageOnPrintf(const char *str) {} #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_WIN_TRACE // Initialize Android logging. Any writes before this are silently lost. void AndroidLogInit(); void SetAbortMessage(const char *); diff --git a/lib/sanitizer_common/sanitizer_common_interceptors.inc b/lib/sanitizer_common/sanitizer_common_interceptors.inc index 50f7837747e6..9f5a91ac99dc 100644 --- a/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interceptors.inc -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -820,16 +819,14 @@ INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { #endif #if SANITIZER_INTERCEPT_MEMCMP - DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc, const void *s1, const void *s2, uptr n, int result) -INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) - return internal_memcmp(a1, a2, size); - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size); +// Common code for `memcmp` and `bcmp`. +int MemcmpInterceptorCommon(void *ctx, + int (*real_fn)(const void *, const void *, uptr), + const void *a1, const void *a2, uptr size) { if (common_flags()->intercept_memcmp) { if (common_flags()->strict_memcmp) { // Check the entire regions even if the first bytes of the buffers are @@ -855,17 +852,39 @@ INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { return r; } } - int result = REAL(memcmp(a1, a2, size)); + int result = real_fn(a1, a2, size); CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(), a1, a2, size, result); return result; } +INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memcmp(a1, a2, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size); + return MemcmpInterceptorCommon(ctx, REAL(memcmp), a1, a2, size); +} + #define INIT_MEMCMP COMMON_INTERCEPT_FUNCTION(memcmp) #else #define INIT_MEMCMP #endif +#if SANITIZER_INTERCEPT_BCMP +INTERCEPTOR(int, bcmp, const void *a1, const void *a2, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memcmp(a1, a2, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, bcmp, a1, a2, size); + return MemcmpInterceptorCommon(ctx, REAL(bcmp), a1, a2, size); +} + +#define INIT_BCMP COMMON_INTERCEPT_FUNCTION(bcmp) +#else +#define INIT_BCMP +#endif + #if SANITIZER_INTERCEPT_MEMCHR INTERCEPTOR(void*, memchr, const void *s, int c, SIZE_T n) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) @@ -1222,7 +1241,8 @@ INTERCEPTOR_WITH_SUFFIX(int, fputs, char *s, void *file) { // libc file streams can call user-supplied functions, see fopencookie. void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fputs, s, file); - COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1); + if (!SANITIZER_MAC || s) + COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1); return REAL(fputs)(s, file); } #define INIT_FPUTS COMMON_INTERCEPT_FUNCTION(fputs) @@ -1235,7 +1255,8 @@ INTERCEPTOR(int, puts, char *s) { // libc file streams can call user-supplied functions, see fopencookie. void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, puts, s); - COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1); + if (!SANITIZER_MAC || s) + COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1); return REAL(puts)(s); } #define INIT_PUTS COMMON_INTERCEPT_FUNCTION(puts) @@ -1818,61 +1839,53 @@ INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) { #define INIT_IOCTL #endif -#if SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS || \ - SANITIZER_INTERCEPT_GETPWENT || SANITIZER_INTERCEPT_FGETPWENT || \ - SANITIZER_INTERCEPT_GETPWENT_R || \ - SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS || \ - SANITIZER_INTERCEPT_FGETPWENT_R || \ - SANITIZER_INTERCEPT_FGETGRENT_R -static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) { +#if SANITIZER_POSIX +UNUSED static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) { if (pwd) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd)); if (pwd->pw_name) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_name, - REAL(strlen)(pwd->pw_name) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_name, + REAL(strlen)(pwd->pw_name) + 1); if (pwd->pw_passwd) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_passwd, - REAL(strlen)(pwd->pw_passwd) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_passwd, + REAL(strlen)(pwd->pw_passwd) + 1); #if !SANITIZER_ANDROID if (pwd->pw_gecos) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_gecos, - REAL(strlen)(pwd->pw_gecos) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos, + REAL(strlen)(pwd->pw_gecos) + 1); #endif -#if SANITIZER_MAC +#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD if (pwd->pw_class) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_class, - REAL(strlen)(pwd->pw_class) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class, + REAL(strlen)(pwd->pw_class) + 1); #endif if (pwd->pw_dir) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_dir, - REAL(strlen)(pwd->pw_dir) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_dir, + REAL(strlen)(pwd->pw_dir) + 1); if (pwd->pw_shell) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(pwd->pw_shell, - REAL(strlen)(pwd->pw_shell) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_shell, + REAL(strlen)(pwd->pw_shell) + 1); } } -static void unpoison_group(void *ctx, __sanitizer_group *grp) { +UNUSED static void unpoison_group(void *ctx, __sanitizer_group *grp) { if (grp) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp, sizeof(*grp)); if (grp->gr_name) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(grp->gr_name, - REAL(strlen)(grp->gr_name) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_name, + REAL(strlen)(grp->gr_name) + 1); if (grp->gr_passwd) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(grp->gr_passwd, - REAL(strlen)(grp->gr_passwd) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_passwd, + REAL(strlen)(grp->gr_passwd) + 1); char **p = grp->gr_mem; for (; *p; ++p) { - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*p, REAL(strlen)(*p) + 1); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1); } - COMMON_INTERCEPTOR_INITIALIZE_RANGE(grp->gr_mem, - (p - grp->gr_mem + 1) * sizeof(*p)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_mem, + (p - grp->gr_mem + 1) * sizeof(*p)); } } -#endif // SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS || - // SANITIZER_INTERCEPT_GETPWENT || SANITIZER_INTERCEPT_FGETPWENT || - // SANITIZER_INTERCEPT_GETPWENT_R || - // SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS +#endif // SANITIZER_POSIX #if SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) { @@ -1881,14 +1894,14 @@ INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) { if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_passwd *res = REAL(getpwnam)(name); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_passwd *, getpwuid, u32 uid) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwuid, uid); __sanitizer_passwd *res = REAL(getpwuid)(uid); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) { @@ -1896,14 +1909,14 @@ INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) { COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name); COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_group *res = REAL(getgrnam)(name); - if (res) unpoison_group(ctx, res); + unpoison_group(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrgid, u32 gid) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getgrgid, gid); __sanitizer_group *res = REAL(getgrgid)(gid); - if (res) unpoison_group(ctx, res); + unpoison_group(ctx, res); return res; } #define INIT_GETPWNAM_AND_FRIENDS \ @@ -1925,10 +1938,8 @@ INTERCEPTOR(int, getpwnam_r, const char *name, __sanitizer_passwd *pwd, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwnam_r)(name, pwd, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_passwd(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_passwd(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1940,10 +1951,8 @@ INTERCEPTOR(int, getpwuid_r, u32 uid, __sanitizer_passwd *pwd, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwuid_r)(uid, pwd, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_passwd(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_passwd(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1956,10 +1965,8 @@ INTERCEPTOR(int, getgrnam_r, const char *name, __sanitizer_group *grp, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrnam_r)(name, grp, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_group(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_group(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1971,10 +1978,8 @@ INTERCEPTOR(int, getgrgid_r, u32 gid, __sanitizer_group *grp, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrgid_r)(gid, grp, buf, buflen, result); - if (!res) { - if (result && *result) unpoison_group(ctx, *result); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && result) + unpoison_group(ctx, *result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; } @@ -1992,14 +1997,14 @@ INTERCEPTOR(__sanitizer_passwd *, getpwent, int dummy) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwent, dummy); __sanitizer_passwd *res = REAL(getpwent)(dummy); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrent, int dummy) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getgrent, dummy); __sanitizer_group *res = REAL(getgrent)(dummy); - if (res) unpoison_group(ctx, res);; + unpoison_group(ctx, res); return res; } #define INIT_GETPWENT \ @@ -2014,14 +2019,14 @@ INTERCEPTOR(__sanitizer_passwd *, fgetpwent, void *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent, fp); __sanitizer_passwd *res = REAL(fgetpwent)(fp); - if (res) unpoison_passwd(ctx, res); + unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, fgetgrent, void *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fgetgrent, fp); __sanitizer_group *res = REAL(fgetgrent)(fp); - if (res) unpoison_group(ctx, res); + unpoison_group(ctx, res); return res; } #define INIT_FGETPWENT \ @@ -2040,10 +2045,8 @@ INTERCEPTOR(int, getpwent_r, __sanitizer_passwd *pwbuf, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwent_r)(pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_passwd(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -2055,10 +2058,8 @@ INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrent_r)(pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_group(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -2078,10 +2079,8 @@ INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_passwd(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -2100,10 +2099,8 @@ INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetgrent_r)(fp, pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } + if (!res && pwbufp) + unpoison_group(ctx, *pwbufp); if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } @@ -3529,13 +3526,16 @@ INTERCEPTOR(SIZE_T, wcrtomb, char *dest, wchar_t src, void *ps) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, wcrtomb, dest, src, ps); if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - SIZE_T res = REAL(wcrtomb)(dest, src, ps); - if (res != ((SIZE_T)-1) && dest) { - SIZE_T write_cnt = res; - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt); + + if (!dest) + return REAL(wcrtomb)(dest, src, ps); + + char local_dest[32]; + SIZE_T res = REAL(wcrtomb)(local_dest, src, ps); + if (res != ((SIZE_T)-1)) { + CHECK_LE(res, sizeof(local_dest)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, res); + REAL(memcpy)(dest, local_dest, res); } return res; } @@ -3545,6 +3545,28 @@ INTERCEPTOR(SIZE_T, wcrtomb, char *dest, wchar_t src, void *ps) { #define INIT_WCRTOMB #endif +#if SANITIZER_INTERCEPT_WCTOMB +INTERCEPTOR(int, wctomb, char *dest, wchar_t src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, wctomb, dest, src); + if (!dest) + return REAL(wctomb)(dest, src); + + char local_dest[32]; + int res = REAL(wctomb)(local_dest, src); + if (res != -1) { + CHECK_LE(res, sizeof(local_dest)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, res); + REAL(memcpy)(dest, local_dest, res); + } + return res; +} + +#define INIT_WCTOMB COMMON_INTERCEPT_FUNCTION(wctomb); +#else +#define INIT_WCTOMB +#endif + #if SANITIZER_INTERCEPT_TCGETATTR INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) { void *ctx; @@ -4041,6 +4063,25 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, #define INIT_SIGPROCMASK #endif +#if SANITIZER_INTERCEPT_PTHREAD_SIGMASK +INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pthread_sigmask, how, set, oldset); + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); + // FIXME: under ASan the call below may write to freed memory and corrupt + // its metadata. See + // https://github.com/google/sanitizers/issues/321. + int res = REAL(pthread_sigmask)(how, set, oldset); + if (!res && oldset) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); + return res; +} +#define INIT_PTHREAD_SIGMASK COMMON_INTERCEPT_FUNCTION(pthread_sigmask); +#else +#define INIT_PTHREAD_SIGMASK +#endif + #if SANITIZER_INTERCEPT_BACKTRACE INTERCEPTOR(int, backtrace, void **buffer, int size) { void *ctx; @@ -4724,6 +4765,20 @@ INTERCEPTOR(char *, tmpnam_r, char *s) { #define INIT_TMPNAM_R #endif +#if SANITIZER_INTERCEPT_TTYNAME +INTERCEPTOR(char *, ttyname, int fd) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ttyname, fd); + char *res = REAL(ttyname)(fd); + if (res != nullptr) + COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); + return res; +} +#define INIT_TTYNAME COMMON_INTERCEPT_FUNCTION(ttyname); +#else +#define INIT_TTYNAME +#endif + #if SANITIZER_INTERCEPT_TTYNAME_R INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) { void *ctx; @@ -5501,12 +5556,21 @@ INTERCEPTOR(void *, __bzero, void *block, uptr size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); } - #define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero); #else #define INIT___BZERO #endif // SANITIZER_INTERCEPT___BZERO +#if SANITIZER_INTERCEPT_BZERO +INTERCEPTOR(void *, bzero, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} +#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero); +#else +#define INIT_BZERO +#endif // SANITIZER_INTERCEPT_BZERO + #if SANITIZER_INTERCEPT_FTIME INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) { void *ctx; @@ -6672,6 +6736,23 @@ INTERCEPTOR(wchar_t *, wcsncat, wchar_t *dst, const wchar_t *src, SIZE_T n) { #define INIT_WCSCAT #endif +#if SANITIZER_INTERCEPT_WCSDUP +INTERCEPTOR(wchar_t *, wcsdup, wchar_t *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, wcsdup, s); + SIZE_T len = REAL(wcslen)(s); + COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (len + 1)); + wchar_t *result = REAL(wcsdup)(s); + if (result) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(wchar_t) * (len + 1)); + return result; +} + +#define INIT_WCSDUP COMMON_INTERCEPT_FUNCTION(wcsdup); +#else +#define INIT_WCSDUP +#endif + #if SANITIZER_INTERCEPT_STRXFRM static SIZE_T RealStrLen(const char *str) { return REAL(strlen)(str); } @@ -9498,6 +9579,7 @@ static void InitializeCommonInterceptors() { INIT_MEMCPY; INIT_MEMCHR; INIT_MEMCMP; + INIT_BCMP; INIT_MEMRCHR; INIT_MEMMEM; INIT_READ; @@ -9578,6 +9660,7 @@ static void InitializeCommonInterceptors() { INIT_WCSTOMBS; INIT_WCSNRTOMBS; INIT_WCRTOMB; + INIT_WCTOMB; INIT_TCGETATTR; INIT_REALPATH; INIT_CANONICALIZE_FILE_NAME; @@ -9599,6 +9682,7 @@ static void InitializeCommonInterceptors() { INIT_SIGSETOPS; INIT_SIGPENDING; INIT_SIGPROCMASK; + INIT_PTHREAD_SIGMASK; INIT_BACKTRACE; INIT__EXIT; INIT_PTHREAD_MUTEX_LOCK; @@ -9637,6 +9721,7 @@ static void InitializeCommonInterceptors() { INIT_PTHREAD_BARRIERATTR_GETPSHARED; INIT_TMPNAM; INIT_TMPNAM_R; + INIT_TTYNAME; INIT_TTYNAME_R; INIT_TEMPNAM; INIT_PTHREAD_SETNAME_NP; @@ -9662,6 +9747,7 @@ static void InitializeCommonInterceptors() { INIT_CAPGET; INIT_AEABI_MEM; INIT___BZERO; + INIT_BZERO; INIT_FTIME; INIT_XDR; INIT_TSEARCH; @@ -9698,6 +9784,7 @@ static void InitializeCommonInterceptors() { INIT_GETLOADAVG; INIT_WCSLEN; INIT_WCSCAT; + INIT_WCSDUP; INIT_WCSXFRM; INIT___WCSXFRM_L; INIT_ACCT; diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_format.inc b/lib/sanitizer_common/sanitizer_common_interceptors_format.inc index 5ebe5a6ba0dd..bbbedda8fbe2 100644 --- a/lib/sanitizer_common/sanitizer_common_interceptors_format.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors_format.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interceptors_format.inc ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc index 2d633c173894..490a04b2181b 100755 --- a/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interceptors_ioctl.inc -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S new file mode 100644 index 000000000000..20f42f1ea94e --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S @@ -0,0 +1,43 @@ +#if defined(__aarch64__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) + +.comm _ZN14__interception10real_vforkE,8,8 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save x30 in the off-stack spill area. + stp xzr, x30, [sp, #-16]! + bl COMMON_INTERCEPTOR_SPILL_AREA + ldp xzr, x30, [sp], 16 + str x30, [x0] + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + adrp x0, _ZN14__interception10real_vforkE + ldr x0, [x0, :lo12:_ZN14__interception10real_vforkE] + blr x0 + + stp x0, xzr, [sp, #-16]! + cmp x0, #0 + b.eq .L_exit + + // x0 != 0 => parent process. Clear stack shadow. + add x0, sp, #16 + bl COMMON_INTERCEPTOR_HANDLE_VFORK + +.L_exit: + // Restore x30. + bl COMMON_INTERCEPTOR_SPILL_AREA + ldr x30, [x0] + ldp x0, xzr, [sp], 16 + + ret +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S new file mode 100644 index 000000000000..780a9d46e26a --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S @@ -0,0 +1,49 @@ +#if defined(__arm__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) + +.comm _ZN14__interception10real_vforkE,4,4 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save LR in the off-stack spill area. + push {r4, lr} + bl COMMON_INTERCEPTOR_SPILL_AREA + pop {r4, lr} + str lr, [r0] + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + ldr r0, .LCPI0_0 +.LPC0_0: + ldr r0, [pc, r0] + mov lr, pc + bx r0 + + push {r0, r4} + cmp r0, #0 + beq .L_exit + + // r0 != 0 => parent process. Clear stack shadow. + add r0, sp, #8 + bl COMMON_INTERCEPTOR_HANDLE_VFORK + +.L_exit: + // Restore LR. + bl COMMON_INTERCEPTOR_SPILL_AREA + ldr lr, [r0] + pop {r0, r4} + + mov pc, lr + +.LCPI0_0: + .long _ZN14__interception10real_vforkE - (.LPC0_0+8) + +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S new file mode 100644 index 000000000000..ed693819c6d4 --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S @@ -0,0 +1,63 @@ +#if defined(__i386__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +.comm _ZN14__interception10real_vforkE,4,4 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Store return address in the spill area and tear down the stack frame. + sub $12, %esp + call COMMON_INTERCEPTOR_SPILL_AREA + mov 12(%esp), %ecx + mov %ecx, (%eax) + add $16, %esp + + call .L0$pb +.L0$pb: + pop %eax +.Ltmp0: + add $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax + call *_ZN14__interception10real_vforkE@GOTOFF(%eax) + + // Restore the stack frame. + // 12(%esp) return address + // 8(%esp) spill %ebx + // 4(%esp) spill REAL(vfork) return value + // (%esp) call frame (arg0) for __*_handle_vfork + sub $16, %esp + mov %ebx, 8(%esp) + mov %eax, 4(%esp) + + // Form GOT address in %ebx. + call .L1$pb +.L1$pb: + pop %ebx +.Ltmp1: + add $_GLOBAL_OFFSET_TABLE_+(.Ltmp1-.L1$pb), %ebx + + // Restore original return address. + call COMMON_INTERCEPTOR_SPILL_AREA + mov (%eax), %ecx + mov %ecx, 12(%esp) + mov 4(%esp), %eax + + // Call handle_vfork in the parent process (%rax != 0). + test %eax, %eax + je .L_exit + + lea 16(%esp), %ecx + mov %ecx, (%esp) + call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT + +.L_exit: + mov 4(%esp), %eax + mov 8(%esp), %ebx + add $12, %esp + ret +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S new file mode 100644 index 000000000000..8147cdd09247 --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S @@ -0,0 +1,41 @@ +#if defined(__x86_64__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +.comm _ZN14__interception10real_vforkE,8,8 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Store return address in the spill area and tear down the stack frame. + push %rcx + call COMMON_INTERCEPTOR_SPILL_AREA + pop %rcx + pop %rdi + mov %rdi, (%rax) + + call *_ZN14__interception10real_vforkE(%rip) + + // Restore return address from the spill area. + push %rcx + push %rax + call COMMON_INTERCEPTOR_SPILL_AREA + mov (%rax), %rdx + mov %rdx, 8(%rsp) + mov (%rsp), %rax + + // Call handle_vfork in the parent process (%rax != 0). + test %rax, %rax + je .L_exit + + lea 16(%rsp), %rdi + call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT + +.L_exit: + pop %rax + ret +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/lib/sanitizer_common/sanitizer_common_interface.inc b/lib/sanitizer_common/sanitizer_common_interface.inc index 377580cb0b4e..c72554973b05 100644 --- a/lib/sanitizer_common/sanitizer_common_interface.inc +++ b/lib/sanitizer_common/sanitizer_common_interface.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interface.inc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Common interface list. diff --git a/lib/sanitizer_common/sanitizer_common_interface_posix.inc b/lib/sanitizer_common/sanitizer_common_interface_posix.inc index bbc725a9d4d1..38f9531148d4 100644 --- a/lib/sanitizer_common/sanitizer_common_interface_posix.inc +++ b/lib/sanitizer_common/sanitizer_common_interface_posix.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_interface_posix.inc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Common interface list only available for Posix systems. diff --git a/lib/sanitizer_common/sanitizer_common_libcdep.cc b/lib/sanitizer_common/sanitizer_common_libcdep.cc index 1c0995b2d27b..363eb4c146ce 100644 --- a/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common_libcdep.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_nolibc.cc b/lib/sanitizer_common/sanitizer_common_nolibc.cc index 5d322907343e..fdd858781216 100644 --- a/lib/sanitizer_common/sanitizer_common_nolibc.cc +++ b/lib/sanitizer_common/sanitizer_common_nolibc.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common_nolibc.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_common_syscalls.inc b/lib/sanitizer_common/sanitizer_common_syscalls.inc index 469c8eb7e149..00bb2aeef111 100644 --- a/lib/sanitizer_common/sanitizer_common_syscalls.inc +++ b/lib/sanitizer_common/sanitizer_common_syscalls.inc @@ -1,9 +1,8 @@ //===-- sanitizer_common_syscalls.inc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc b/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc index 9ff8798a5731..2c7180122148 100644 --- a/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc +++ b/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_fuchsia.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -133,7 +132,7 @@ class TracePcGuardController final { // The first sample goes at [1] to reserve [0] for the magic number. next_index_ = 1 + num_guards; - zx_status_t status = _zx_vmo_create(DataSize(), 0, &vmo_); + zx_status_t status = _zx_vmo_create(DataSize(), ZX_VMO_RESIZABLE, &vmo_); CHECK_EQ(status, ZX_OK); // Give the VMO a name including our process KOID so it's easy to spot. diff --git a/lib/sanitizer_common/sanitizer_coverage_interface.inc b/lib/sanitizer_common/sanitizer_coverage_interface.inc index f909b74c9752..7beeff7e8af8 100644 --- a/lib/sanitizer_common/sanitizer_coverage_interface.inc +++ b/lib/sanitizer_common/sanitizer_coverage_interface.inc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_interface.inc ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Coverage interface list. diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc index bea277587289..9dbf2eb52978 100644 --- a/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc +++ b/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_libcdep_new.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Sanitizer Coverage Controller for Trace PC Guard. diff --git a/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc b/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc index d5e459f2c020..1f0f69dc43eb 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_dll_thunk.cc -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc b/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc index 988a2065ffa9..2a4199756f3b 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_dynamic_runtime_thunk.cc -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,3 +18,9 @@ #define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name) #include "sanitizer_coverage_interface.inc" #endif // SANITIZER_DYNAMIC_RUNTIME_THUNK + +namespace __sanitizer { +// Add one, otherwise unused, external symbol to this object file so that the +// Visual C++ linker includes it and reads the .drective section. +void ForceWholeArchiveIncludeForSanCov() {} +} diff --git a/lib/sanitizer_common/sanitizer_coverage_win_sections.cc b/lib/sanitizer_common/sanitizer_coverage_win_sections.cc index 108f76effeeb..403d46c8c4f1 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_sections.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_sections.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_sections.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,35 +26,40 @@ #include "sanitizer_platform.h" #if SANITIZER_WINDOWS #include <stdint.h> -extern "C" { -// The Guard array and counter array should both be merged into the .data -// section to reduce the number of PE sections However, because PCTable is -// constant it should be merged with the .rdata section. -#pragma section(".SCOV$GA", read, write) // NOLINT -// Use align(1) to avoid adding any padding that will mess up clients trying to -// determine the start and end of the array. -__declspec(allocate(".SCOV$GA")) __declspec(align(1)) uint64_t - __start___sancov_guards = 0; -#pragma section(".SCOV$GZ", read, write) // NOLINT -__declspec(allocate(".SCOV$GZ")) __declspec(align(1)) uint64_t - __stop___sancov_guards = 0; +extern "C" { +// Use uint64_t so the linker won't need to add any padding if it tries to word +// align the start of the 8-bit counters array. The array will always start 8 +// bytes after __start_sancov_cntrs. #pragma section(".SCOV$CA", read, write) // NOLINT -__declspec(allocate(".SCOV$CA")) __declspec(align(1)) uint64_t - __start___sancov_cntrs = 0; +__declspec(allocate(".SCOV$CA")) uint64_t __start___sancov_cntrs = 0; + +// Even though we said not to align __stop__sancov_cntrs (using the "align" +// declspec), MSVC's linker may try to align the section, .SCOV$CZ, containing +// it. This can cause a mismatch between the number of PCs and counters since +// each PCTable element is 8 bytes (unlike counters which are 1 byte) so no +// padding would be added to align .SCOVP$Z, However, if .SCOV$CZ section is 1 +// byte, the linker won't try to align it on an 8-byte boundary, so use a +// uint8_t for __stop_sancov_cntrs. #pragma section(".SCOV$CZ", read, write) // NOLINT -__declspec(allocate(".SCOV$CZ")) __declspec(align(1)) uint64_t +__declspec(allocate(".SCOV$CZ")) __declspec(align(1)) uint8_t __stop___sancov_cntrs = 0; +#pragma section(".SCOV$GA", read, write) // NOLINT +__declspec(allocate(".SCOV$GA")) uint64_t __start___sancov_guards = 0; +#pragma section(".SCOV$GZ", read, write) // NOLINT +__declspec(allocate(".SCOV$GZ")) __declspec(align(1)) uint8_t + __stop___sancov_guards = 0; + +// The guard array and counter array should both be merged into the .data +// section to reduce the number of PE sections. However, because PCTable is +// constant it should be merged with the .rdata section. #pragma comment(linker, "/MERGE:.SCOV=.data") -// Use uint64_t so there won't be any issues if the linker tries to word align -// the pc array. #pragma section(".SCOVP$A", read) // NOLINT -__declspec(allocate(".SCOVP$A")) __declspec(align(1)) uint64_t - __start___sancov_pcs = 0; +__declspec(allocate(".SCOVP$A")) uint64_t __start___sancov_pcs = 0; #pragma section(".SCOVP$Z", read) // NOLINT -__declspec(allocate(".SCOVP$Z")) __declspec(align(1)) uint64_t +__declspec(allocate(".SCOVP$Z")) __declspec(align(1)) uint8_t __stop___sancov_pcs = 0; #pragma comment(linker, "/MERGE:.SCOVP=.rdata") diff --git a/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc b/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc index 0926f460b3cd..1fd10d5b83a1 100644 --- a/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc +++ b/lib/sanitizer_common/sanitizer_coverage_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- sanitizer_coverage_win_weak_interception.cc -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in Sanitizer Coverage when it implemented as a diff --git a/lib/sanitizer_common/sanitizer_dbghelp.h b/lib/sanitizer_common/sanitizer_dbghelp.h index 1689edbf92db..00a539980067 100644 --- a/lib/sanitizer_common/sanitizer_dbghelp.h +++ b/lib/sanitizer_common/sanitizer_dbghelp.h @@ -1,9 +1,8 @@ //===-- sanitizer_dbghelp.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector.h b/lib/sanitizer_common/sanitizer_deadlock_detector.h index 86d5743e9794..b80cff460eda 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector.h +++ b/lib/sanitizer_common/sanitizer_deadlock_detector.h @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,8 +25,8 @@ #ifndef SANITIZER_DEADLOCK_DETECTOR_H #define SANITIZER_DEADLOCK_DETECTOR_H -#include "sanitizer_common.h" #include "sanitizer_bvgraph.h" +#include "sanitizer_common.h" namespace __sanitizer { @@ -58,7 +57,6 @@ class DeadlockDetectorTLS { // Returns true if this is the first (non-recursive) acquisition of this lock. bool addLock(uptr lock_id, uptr current_epoch, u32 stk) { - // Printf("addLock: %zx %zx stk %u\n", lock_id, current_epoch, stk); CHECK_EQ(epoch_, current_epoch); if (!bv_.setBit(lock_id)) { // The lock is already held by this thread, it must be recursive. @@ -84,7 +82,6 @@ class DeadlockDetectorTLS { } } } - // Printf("remLock: %zx %zx\n", lock_id, epoch_); if (!bv_.clearBit(lock_id)) return; // probably addLock happened before flush if (n_all_locks_) { @@ -158,7 +155,6 @@ class DeadlockDetector { if (!available_nodes_.empty()) return getAvailableNode(data); if (!recycled_nodes_.empty()) { - // Printf("recycling: n_edges_ %zd\n", n_edges_); for (sptr i = n_edges_ - 1; i >= 0; i--) { if (recycled_nodes_.getBit(edges_[i].from) || recycled_nodes_.getBit(edges_[i].to)) { @@ -255,8 +251,6 @@ class DeadlockDetector { unique_tid}; edges_[n_edges_++] = e; } - // Printf("Edge%zd: %u %zd=>%zd in T%d\n", - // n_edges_, stk, added_edges[i], cur_idx, unique_tid); } return n_added_edges; } diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector1.cc b/lib/sanitizer_common/sanitizer_deadlock_detector1.cc index 68a99d2c6268..a151a190c180 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector1.cc +++ b/lib/sanitizer_common/sanitizer_deadlock_detector1.cc @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector1.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector2.cc b/lib/sanitizer_common/sanitizer_deadlock_detector2.cc index 0b0085a48bd5..ed9ea26d3902 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector2.cc +++ b/lib/sanitizer_common/sanitizer_deadlock_detector2.cc @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector2.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h b/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h index 11674dff5397..a4722b080ebd 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h +++ b/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h @@ -1,9 +1,8 @@ //===-- sanitizer_deadlock_detector_interface.h -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_errno.cc b/lib/sanitizer_common/sanitizer_errno.cc index a6f9fc6125eb..1600de566393 100644 --- a/lib/sanitizer_common/sanitizer_errno.cc +++ b/lib/sanitizer_common/sanitizer_errno.cc @@ -1,9 +1,8 @@ //===-- sanitizer_errno.cc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_errno.h b/lib/sanitizer_common/sanitizer_errno.h index 9322608aa8f5..584e66e4a861 100644 --- a/lib/sanitizer_common/sanitizer_errno.h +++ b/lib/sanitizer_common/sanitizer_errno.h @@ -1,9 +1,8 @@ //===-- sanitizer_errno.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_errno_codes.h b/lib/sanitizer_common/sanitizer_errno_codes.h index dba774c5b6c5..f388d0d36463 100644 --- a/lib/sanitizer_common/sanitizer_errno_codes.h +++ b/lib/sanitizer_common/sanitizer_errno_codes.h @@ -1,9 +1,8 @@ //===-- sanitizer_errno_codes.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_file.cc b/lib/sanitizer_common/sanitizer_file.cc index 278d75c520ae..3cb4974bdab2 100644 --- a/lib/sanitizer_common/sanitizer_file.cc +++ b/lib/sanitizer_common/sanitizer_file.cc @@ -1,9 +1,8 @@ //===-- sanitizer_file.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_file.h b/lib/sanitizer_common/sanitizer_file.h index 52e6ef8fb1ac..4a78a0e0ac88 100644 --- a/lib/sanitizer_common/sanitizer_file.h +++ b/lib/sanitizer_common/sanitizer_file.h @@ -1,9 +1,8 @@ //===-- sanitizer_file.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_flag_parser.cc b/lib/sanitizer_common/sanitizer_flag_parser.cc index 67830b2940bb..e380d3b55e36 100644 --- a/lib/sanitizer_common/sanitizer_flag_parser.cc +++ b/lib/sanitizer_common/sanitizer_flag_parser.cc @@ -1,9 +1,8 @@ //===-- sanitizer_flag_parser.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -63,7 +62,7 @@ void FlagParser::PrintFlagDescriptions() { } void FlagParser::fatal_error(const char *err) { - Printf("ERROR: %s\n", err); + Printf("%s: ERROR: %s\n", SanitizerToolName, err); Die(); } @@ -76,10 +75,17 @@ void FlagParser::skip_whitespace() { while (is_space(buf_[pos_])) ++pos_; } -void FlagParser::parse_flag() { +void FlagParser::parse_flag(const char *env_option_name) { uptr name_start = pos_; while (buf_[pos_] != 0 && buf_[pos_] != '=' && !is_space(buf_[pos_])) ++pos_; - if (buf_[pos_] != '=') fatal_error("expected '='"); + if (buf_[pos_] != '=') { + if (env_option_name) { + Printf("%s: ERROR: expected '=' in %s\n", SanitizerToolName, + env_option_name); + Die(); + } else + fatal_error("expected '='"); + } char *name = ll_strndup(buf_ + name_start, pos_ - name_start); uptr value_start = ++pos_; @@ -101,11 +107,11 @@ void FlagParser::parse_flag() { if (!res) fatal_error("Flag parsing failed."); } -void FlagParser::parse_flags() { +void FlagParser::parse_flags(const char *env_option_name) { while (true) { skip_whitespace(); if (buf_[pos_] == 0) break; - parse_flag(); + parse_flag(env_option_name); } // Do a sanity check for certain flags. @@ -113,7 +119,13 @@ void FlagParser::parse_flags() { common_flags_dont_use.malloc_context_size = 1; } -void FlagParser::ParseString(const char *s) { +void FlagParser::ParseStringFromEnv(const char *env_name) { + const char *env = GetEnv(env_name); + VPrintf(1, "%s: %s\n", env_name, env ? env : "<empty>"); + ParseString(env, env_name); +} + +void FlagParser::ParseString(const char *s, const char *env_option_name) { if (!s) return; // Backup current parser state to allow nested ParseString() calls. const char *old_buf_ = buf_; @@ -121,7 +133,7 @@ void FlagParser::ParseString(const char *s) { buf_ = s; pos_ = 0; - parse_flags(); + parse_flags(env_option_name); buf_ = old_buf_; pos_ = old_pos_; @@ -140,7 +152,7 @@ bool FlagParser::ParseFile(const char *path, bool ignore_missing) { Printf("Failed to read options from '%s': error %d\n", path, err); return false; } - ParseString(data); + ParseString(data, path); UnmapOrDie(data, data_mapped_size); return true; } diff --git a/lib/sanitizer_common/sanitizer_flag_parser.h b/lib/sanitizer_common/sanitizer_flag_parser.h index 705bfe23b309..8e12700bbe8c 100644 --- a/lib/sanitizer_common/sanitizer_flag_parser.h +++ b/lib/sanitizer_common/sanitizer_flag_parser.h @@ -1,9 +1,8 @@ //===-- sanitizer_flag_parser.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,6 +22,9 @@ namespace __sanitizer { class FlagHandlerBase { public: virtual bool Parse(const char *value) { return false; } + + protected: + ~FlagHandlerBase() {}; }; template <typename T> @@ -97,6 +99,15 @@ inline bool FlagHandler<uptr>::Parse(const char *value) { return ok; } +template <> +inline bool FlagHandler<s64>::Parse(const char *value) { + const char *value_end; + *t_ = internal_simple_strtoll(value, &value_end, 10); + bool ok = *value_end == 0; + if (!ok) Printf("ERROR: Invalid value for s64 option: '%s'\n", value); + return ok; +} + class FlagParser { static const int kMaxFlags = 200; struct Flag { @@ -113,7 +124,8 @@ class FlagParser { FlagParser(); void RegisterHandler(const char *name, FlagHandlerBase *handler, const char *desc); - void ParseString(const char *s); + void ParseString(const char *s, const char *env_name = 0); + void ParseStringFromEnv(const char *env_name); bool ParseFile(const char *path, bool ignore_missing); void PrintFlagDescriptions(); @@ -123,8 +135,8 @@ class FlagParser { void fatal_error(const char *err); bool is_space(char c); void skip_whitespace(); - void parse_flags(); - void parse_flag(); + void parse_flags(const char *env_option_name); + void parse_flag(const char *env_option_name); bool run_handler(const char *name, const char *value); char *ll_strndup(const char *s, uptr n); }; diff --git a/lib/sanitizer_common/sanitizer_flags.cc b/lib/sanitizer_common/sanitizer_flags.cc index 4631dfb7120c..c587b1884935 100644 --- a/lib/sanitizer_common/sanitizer_flags.cc +++ b/lib/sanitizer_common/sanitizer_flags.cc @@ -1,9 +1,8 @@ //===-- sanitizer_flags.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_flags.h b/lib/sanitizer_common/sanitizer_flags.h index c2ec29d1656e..8f5e987da3ff 100644 --- a/lib/sanitizer_common/sanitizer_flags.h +++ b/lib/sanitizer_common/sanitizer_flags.h @@ -1,9 +1,8 @@ //===-- sanitizer_flags.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_flags.inc b/lib/sanitizer_common/sanitizer_flags.inc index b047741b9083..7d592bdcb61f 100644 --- a/lib/sanitizer_common/sanitizer_flags.inc +++ b/lib/sanitizer_common/sanitizer_flags.inc @@ -1,9 +1,8 @@ //===-- sanitizer_flags.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -219,9 +218,9 @@ COMMON_FLAG(bool, intercept_stat, true, COMMON_FLAG(bool, intercept_send, true, "If set, uses custom wrappers for send* functions " "to find more errors.") -COMMON_FLAG(bool, decorate_proc_maps, false, "If set, decorate sanitizer " - "mappings in /proc/self/maps with " - "user-readable names") +COMMON_FLAG(bool, decorate_proc_maps, (bool)SANITIZER_ANDROID, + "If set, decorate sanitizer mappings in /proc/self/maps with " + "user-readable names") COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool " "found an error") COMMON_FLAG( diff --git a/lib/sanitizer_common/sanitizer_freebsd.h b/lib/sanitizer_common/sanitizer_freebsd.h index c9bba8064af0..64cb21f1c3d2 100644 --- a/lib/sanitizer_common/sanitizer_freebsd.h +++ b/lib/sanitizer_common/sanitizer_freebsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_freebsd.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_fuchsia.cc b/lib/sanitizer_common/sanitizer_fuchsia.cc index 2c259b6cd7da..9c032fa8995c 100644 --- a/lib/sanitizer_common/sanitizer_fuchsia.cc +++ b/lib/sanitizer_common/sanitizer_fuchsia.cc @@ -1,9 +1,8 @@ //===-- sanitizer_fuchsia.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -47,9 +46,14 @@ unsigned int internal_sleep(unsigned int seconds) { return 0; } -u64 NanoTime() { return _zx_clock_get(ZX_CLOCK_UTC); } +u64 NanoTime() { + zx_time_t time; + zx_status_t status = _zx_clock_get(ZX_CLOCK_UTC, &time); + CHECK_EQ(status, ZX_OK); + return time; +} -u64 MonotonicNanoTime() { return _zx_clock_get(ZX_CLOCK_MONOTONIC); } +u64 MonotonicNanoTime() { return _zx_clock_get_monotonic(); } uptr internal_getpid() { zx_info_handle_basic_t info; @@ -253,12 +257,14 @@ static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size, return addr; } -uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size) { +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size, + const char *name) { return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, false); } -uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size) { +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size, + const char *name) { return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, true); } diff --git a/lib/sanitizer_common/sanitizer_fuchsia.h b/lib/sanitizer_common/sanitizer_fuchsia.h index 18821b4fd9f1..5a2ad32b4113 100644 --- a/lib/sanitizer_common/sanitizer_fuchsia.h +++ b/lib/sanitizer_common/sanitizer_fuchsia.h @@ -1,9 +1,8 @@ //===-- sanitizer_fuchsia.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_getauxval.h b/lib/sanitizer_common/sanitizer_getauxval.h index b22fcade5f9d..cbd1af12c04f 100644 --- a/lib/sanitizer_common/sanitizer_getauxval.h +++ b/lib/sanitizer_common/sanitizer_getauxval.h @@ -1,9 +1,8 @@ //===-- sanitizer_getauxval.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_hash.h b/lib/sanitizer_common/sanitizer_hash.h new file mode 100644 index 000000000000..3d97dcc5d280 --- /dev/null +++ b/lib/sanitizer_common/sanitizer_hash.h @@ -0,0 +1,43 @@ +//===-- sanitizer_common.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a simple hash function. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_HASH_H +#define SANITIZER_HASH_H + +#include "sanitizer_internal_defs.h" + +namespace __sanitizer { +class MurMur2HashBuilder { + static const u32 m = 0x5bd1e995; + static const u32 seed = 0x9747b28c; + static const u32 r = 24; + u32 h; + + public: + explicit MurMur2HashBuilder(u32 init = 0) { h = seed ^ init; } + void add(u32 k) { + k *= m; + k ^= k >> r; + k *= m; + h *= m; + h ^= k; + } + u32 get() { + u32 x = h; + x ^= x >> 13; + x *= m; + x ^= x >> 15; + return x; + } +}; +} //namespace __sanitizer + +#endif // SANITIZER_HASH_H diff --git a/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc index 86cb44047606..f29226b3ee3a 100644 --- a/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc +++ b/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc @@ -1,9 +1,8 @@ //===-- sanitizer_interceptors_ioctl_netbsd.inc -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,7 @@ struct ioctl_desc { const char *name; }; -const unsigned ioctl_table_max = 1202; +const unsigned ioctl_table_max = 1200; static ioctl_desc ioctl_table[ioctl_table_max]; static unsigned ioctl_table_size = 0; @@ -298,9 +297,6 @@ static void ioctl_table_fill() { _(IRFRAMETTY_GET_DEVICE, WRITE, sizeof(unsigned int)); _(IRFRAMETTY_GET_DONGLE, WRITE, sizeof(unsigned int)); _(IRFRAMETTY_SET_DONGLE, READ, sizeof(unsigned int)); - /* Entries from file: dev/isa/satlinkio.h */ - _(SATIORESET, NONE, 0); - _(SATIOGID, WRITE, struct_satlink_id_sz); /* Entries from file: dev/isa/isvio.h */ _(ISV_CMD, READWRITE, struct_isv_cmd_sz); /* Entries from file: dev/isa/wtreg.h */ @@ -649,8 +645,8 @@ static void ioctl_table_fill() { _(SPKRTUNE, NONE, 0); _(SPKRGETVOL, WRITE, sizeof(unsigned int)); _(SPKRSETVOL, READ, sizeof(unsigned int)); - /* Entries from file: dev/nvmm/nvmm_ioctl.h */ #if 0 /* WIP */ + /* Entries from file: dev/nvmm/nvmm_ioctl.h */ _(NVMM_IOC_CAPABILITY, WRITE, struct_nvmm_ioc_capability_sz); _(NVMM_IOC_MACHINE_CREATE, READWRITE, struct_nvmm_ioc_machine_create_sz); _(NVMM_IOC_MACHINE_DESTROY, READ, struct_nvmm_ioc_machine_destroy_sz); @@ -659,7 +655,7 @@ static void ioctl_table_fill() { _(NVMM_IOC_VCPU_DESTROY, READ, struct_nvmm_ioc_vcpu_destroy_sz); _(NVMM_IOC_VCPU_SETSTATE, READ, struct_nvmm_ioc_vcpu_setstate_sz); _(NVMM_IOC_VCPU_GETSTATE, READ, struct_nvmm_ioc_vcpu_getstate_sz); - _(NVMM_IOC_VCPU_INJECT, READWRITE, struct_nvmm_ioc_vcpu_inject_sz); + _(NVMM_IOC_VCPU_INJECT, READ, struct_nvmm_ioc_vcpu_inject_sz); _(NVMM_IOC_VCPU_RUN, READWRITE, struct_nvmm_ioc_vcpu_run_sz); _(NVMM_IOC_GPA_MAP, READ, struct_nvmm_ioc_gpa_map_sz); _(NVMM_IOC_GPA_UNMAP, READ, struct_nvmm_ioc_gpa_unmap_sz); diff --git a/lib/sanitizer_common/sanitizer_interface_internal.h b/lib/sanitizer_common/sanitizer_interface_internal.h index f53d9557eab4..c110eff130f2 100644 --- a/lib/sanitizer_common/sanitizer_interface_internal.h +++ b/lib/sanitizer_common/sanitizer_interface_internal.h @@ -1,9 +1,8 @@ //===-- sanitizer_interface_internal.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_internal_defs.h b/lib/sanitizer_common/sanitizer_internal_defs.h index 14258d617e62..e0c6506bed51 100644 --- a/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/lib/sanitizer_common/sanitizer_internal_defs.h @@ -1,9 +1,8 @@ //===-- sanitizer_internal_defs.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -423,7 +422,6 @@ inline void Trap() { namespace __asan { using namespace __sanitizer; } // NOLINT namespace __dsan { using namespace __sanitizer; } // NOLINT namespace __dfsan { using namespace __sanitizer; } // NOLINT -namespace __esan { using namespace __sanitizer; } // NOLINT namespace __lsan { using namespace __sanitizer; } // NOLINT namespace __msan { using namespace __sanitizer; } // NOLINT namespace __hwasan { using namespace __sanitizer; } // NOLINT diff --git a/lib/sanitizer_common/sanitizer_lfstack.h b/lib/sanitizer_common/sanitizer_lfstack.h index 879cc80921c7..af2ca55ec3f6 100644 --- a/lib/sanitizer_common/sanitizer_lfstack.h +++ b/lib/sanitizer_common/sanitizer_lfstack.h @@ -1,9 +1,8 @@ //===-- sanitizer_lfstack.h -=-----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_libc.cc b/lib/sanitizer_common/sanitizer_libc.cc index 4032cb10461f..95c74441fd21 100644 --- a/lib/sanitizer_common/sanitizer_libc.cc +++ b/lib/sanitizer_common/sanitizer_libc.cc @@ -1,9 +1,8 @@ //===-- sanitizer_libc.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_libc.h b/lib/sanitizer_common/sanitizer_libc.h index 7ed4e28a5695..3d5db35d68ba 100644 --- a/lib/sanitizer_common/sanitizer_libc.h +++ b/lib/sanitizer_common/sanitizer_libc.h @@ -1,9 +1,8 @@ //===-- sanitizer_libc.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_libignore.cc b/lib/sanitizer_common/sanitizer_libignore.cc index 49c4ba429492..6c7c132e99eb 100644 --- a/lib/sanitizer_common/sanitizer_libignore.cc +++ b/lib/sanitizer_common/sanitizer_libignore.cc @@ -1,9 +1,8 @@ //===-- sanitizer_libignore.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/sanitizer_common/sanitizer_libignore.h b/lib/sanitizer_common/sanitizer_libignore.h index 49967b1e8d2a..256f685979f4 100644 --- a/lib/sanitizer_common/sanitizer_libignore.h +++ b/lib/sanitizer_common/sanitizer_libignore.h @@ -1,9 +1,8 @@ //===-- sanitizer_libignore.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc index 48795674c54c..88ab0979bb05 100644 --- a/lib/sanitizer_common/sanitizer_linux.cc +++ b/lib/sanitizer_common/sanitizer_linux.cc @@ -1,9 +1,8 @@ //===-- sanitizer_linux.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -129,12 +128,6 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0 #endif -#if defined(__x86_64__) || SANITIZER_MIPS64 -extern "C" { -extern void internal_sigreturn(); -} -#endif - // Note : FreeBSD had implemented both // Linux and OpenBSD apis, available from // future 12.x version most likely @@ -401,7 +394,7 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) { return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, bufsize); #else - return internal_syscall(SYSCALL(readlink), path, buf, bufsize); + return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize); #endif } @@ -838,24 +831,6 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { } return result; } - -// Invokes sigaction via a raw syscall with a restorer, but does not support -// all platforms yet. -// We disable for Go simply because we have not yet added to buildgo.sh. -#if (defined(__x86_64__) || SANITIZER_MIPS64) && !SANITIZER_GO -int internal_sigaction_syscall(int signum, const void *act, void *oldact) { - if (act == nullptr) - return internal_sigaction_norestorer(signum, act, oldact); - __sanitizer_sigaction u_adjust; - internal_memcpy(&u_adjust, act, sizeof(u_adjust)); -#if !SANITIZER_ANDROID || !SANITIZER_MIPS32 - if (u_adjust.sa_restorer == nullptr) { - u_adjust.sa_restorer = internal_sigreturn; - } -#endif - return internal_sigaction_norestorer(signum, (const void *)&u_adjust, oldact); -} -#endif // defined(__x86_64__) && !SANITIZER_GO #endif // SANITIZER_LINUX uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, @@ -1055,6 +1030,8 @@ uptr GetMaxVirtualAddress() { return (1ULL << 40) - 1; // 0x000000ffffffffffUL; # elif defined(__s390x__) return (1ULL << 53) - 1; // 0x001fffffffffffffUL; +#elif defined(__sparc__) + return ~(uptr)0; # else return (1ULL << 47) - 1; // 0x00007fffffffffffUL; # endif @@ -1846,10 +1823,20 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { u64 esr; if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN; return esr & ESR_ELx_WNR ? WRITE : READ; -#elif SANITIZER_SOLARIS && defined(__sparc__) +#elif defined(__sparc__) // Decode the instruction to determine the access type. // From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype). +#if SANITIZER_SOLARIS uptr pc = ucontext->uc_mcontext.gregs[REG_PC]; +#else + // Historical BSDism here. + struct sigcontext *scontext = (struct sigcontext *)context; +#if defined(__arch64__) + uptr pc = scontext->sigc_regs.tpc; +#else + uptr pc = scontext->si_regs.pc; +#endif +#endif u32 instr = *(u32 *)pc; return (instr >> 21) & 1 ? WRITE: READ; #else @@ -1940,28 +1927,27 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { // pointer, but GCC always uses r31 when we need a frame pointer. *bp = ucontext->uc_mcontext.regs->gpr[PT_R31]; #elif defined(__sparc__) - ucontext_t *ucontext = (ucontext_t*)context; - uptr *stk_ptr; -# if defined(__sparcv9) || defined (__arch64__) -# ifndef MC_PC -# define MC_PC REG_PC -# endif -# ifndef MC_O6 -# define MC_O6 REG_O6 +#if defined(__arch64__) || defined(__sparcv9) +#define STACK_BIAS 2047 +#else +#define STACK_BIAS 0 # endif # if SANITIZER_SOLARIS -# define mc_gregs gregs -# endif - *pc = ucontext->uc_mcontext.mc_gregs[MC_PC]; - *sp = ucontext->uc_mcontext.mc_gregs[MC_O6]; - stk_ptr = (uptr *) (*sp + 2047); - *bp = stk_ptr[15]; -# else + ucontext_t *ucontext = (ucontext_t *)context; *pc = ucontext->uc_mcontext.gregs[REG_PC]; - *sp = ucontext->uc_mcontext.gregs[REG_O6]; - stk_ptr = (uptr *) *sp; - *bp = stk_ptr[15]; + *sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS; +#else + // Historical BSDism here. + struct sigcontext *scontext = (struct sigcontext *)context; +#if defined(__arch64__) + *pc = scontext->sigc_regs.tpc; + *sp = scontext->sigc_regs.u_regs[14] + STACK_BIAS; +#else + *pc = scontext->si_regs.pc; + *sp = scontext->si_regs.u_regs[14]; +#endif # endif + *bp = (uptr)((uhwptr *)*sp)[14] + STACK_BIAS; #elif defined(__mips__) ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.pc; diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h index c309e33f81b1..c28347ad963a 100644 --- a/lib/sanitizer_common/sanitizer_linux.h +++ b/lib/sanitizer_common/sanitizer_linux.h @@ -1,9 +1,8 @@ //===-- sanitizer_linux.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -59,10 +58,6 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5); // (like the process-wide error reporting SEGV handler) must use // internal_sigaction instead. int internal_sigaction_norestorer(int signum, const void *act, void *oldact); -#if (defined(__x86_64__) || SANITIZER_MIPS64) && !SANITIZER_GO -// Uses a raw system call to avoid interceptors. -int internal_sigaction_syscall(int signum, const void *act, void *oldact); -#endif void internal_sigdelset(__sanitizer_sigset_t *set, int signum); #if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \ || defined(__powerpc64__) || defined(__s390__) || defined(__i386__) \ @@ -72,6 +67,9 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, #endif #elif SANITIZER_FREEBSD void internal_sigdelset(__sanitizer_sigset_t *set, int signum); +#elif SANITIZER_NETBSD +void internal_sigdelset(__sanitizer_sigset_t *set, int signum); +uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg); #endif // SANITIZER_LINUX // This class reads thread IDs from /proc/<pid>/task using only syscalls. @@ -106,6 +104,17 @@ bool LibraryNameIs(const char *full_name, const char *base_name); // Call cb for each region mapped by map. void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)); +// Releases memory pages entirely within the [beg, end] address range. +// The pages no longer count toward RSS; reads are guaranteed to return 0. +// Requires (but does not verify!) that pages are MAP_PRIVATE. +INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) { + // man madvise on Linux promises zero-fill for anonymous private pages. + // Testing shows the same behaviour for private (but not anonymous) mappings + // of shm_open() files, as long as the underlying file is untouched. + CHECK(SANITIZER_LINUX); + ReleaseMemoryPagesToOS(beg, end); +} + #if SANITIZER_ANDROID #if defined(__aarch64__) diff --git a/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 6ce47ec620dc..0608898a1464 100644 --- a/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_linux_libcdep.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -53,6 +52,7 @@ #endif #if SANITIZER_SOLARIS +#include <stdlib.h> #include <thread.h> #endif diff --git a/lib/sanitizer_common/sanitizer_linux_mips64.S b/lib/sanitizer_common/sanitizer_linux_mips64.S deleted file mode 100644 index 8729642aa654..000000000000 --- a/lib/sanitizer_common/sanitizer_linux_mips64.S +++ /dev/null @@ -1,23 +0,0 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. - -// Avoid being marked as needing an executable stack: -#if defined(__linux__) && defined(__ELF__) -.section .note.GNU-stack,"",%progbits -#endif - -// Further contents are mips64 only: -#if defined(__linux__) && defined(__mips64) - -.section .text -.set noreorder -.globl internal_sigreturn -.type internal_sigreturn, @function -internal_sigreturn: - - li $v0,5211 // #5211 is for SYS_rt_sigreturn - syscall - -.size internal_sigreturn, .-internal_sigreturn - -#endif // defined(__linux__) && defined(__mips64) diff --git a/lib/sanitizer_common/sanitizer_linux_s390.cc b/lib/sanitizer_common/sanitizer_linux_s390.cc index ad8c87f2d6b6..b681bef35b76 100644 --- a/lib/sanitizer_common/sanitizer_linux_s390.cc +++ b/lib/sanitizer_common/sanitizer_linux_s390.cc @@ -1,9 +1,8 @@ //===-- sanitizer_linux_s390.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_linux_x86_64.S b/lib/sanitizer_common/sanitizer_linux_x86_64.S deleted file mode 100644 index 8ff909542b68..000000000000 --- a/lib/sanitizer_common/sanitizer_linux_x86_64.S +++ /dev/null @@ -1,25 +0,0 @@ -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. - -// Avoid being marked as needing an executable stack: -#if defined(__linux__) && defined(__ELF__) -.section .note.GNU-stack,"",%progbits -#endif - -// Further contents are x86_64-only: -#if defined(__linux__) && defined(__x86_64__) - -#include "../builtins/assembly.h" - -// If the "naked" function attribute were supported for x86 we could -// do this via inline asm. -.text -.balign 4 -DEFINE_COMPILERRT_FUNCTION(internal_sigreturn) - mov $0xf, %eax // 0xf == SYS_rt_sigreturn - mov %rcx, %r10 - syscall - ret // Won't normally reach here. -END_COMPILERRT_FUNCTION(internal_sigreturn) - -#endif // defined(__linux__) && defined(__x86_64__) diff --git a/lib/sanitizer_common/sanitizer_list.h b/lib/sanitizer_common/sanitizer_list.h index 598ce51d848b..f0b925945e66 100644 --- a/lib/sanitizer_common/sanitizer_list.h +++ b/lib/sanitizer_common/sanitizer_list.h @@ -1,9 +1,8 @@ //===-- sanitizer_list.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_local_address_space_view.h b/lib/sanitizer_common/sanitizer_local_address_space_view.h index ec1847abc53d..5d1b5264b5ed 100644 --- a/lib/sanitizer_common/sanitizer_local_address_space_view.h +++ b/lib/sanitizer_common/sanitizer_local_address_space_view.h @@ -1,9 +1,8 @@ //===-- sanitizer_local_address_space_view.h --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc index 8954a7a88073..d9b7c4b12926 100644 --- a/lib/sanitizer_common/sanitizer_mac.cc +++ b/lib/sanitizer_common/sanitizer_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_mac.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -240,25 +239,25 @@ int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, (size_t)newlen); } -int internal_forkpty(int *amaster) { - int master, slave; - if (openpty(&master, &slave, nullptr, nullptr, nullptr) == -1) return -1; +int internal_forkpty(int *aparent) { + int parent, worker; + if (openpty(&parent, &worker, nullptr, nullptr, nullptr) == -1) return -1; int pid = internal_fork(); if (pid == -1) { - close(master); - close(slave); + close(parent); + close(worker); return -1; } if (pid == 0) { - close(master); - if (login_tty(slave) != 0) { + close(parent); + if (login_tty(worker) != 0) { // We already forked, there's not much we can do. Let's quit. Report("login_tty failed (errno %d)\n", errno); internal__exit(1); } } else { - *amaster = master; - close(slave); + *aparent = parent; + close(worker); } return pid; } @@ -559,8 +558,8 @@ MacosVersion GetMacosVersionInternal() { if (minor >= 5) return MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4; return MACOS_VERSION_HIGH_SIERRA; - case 18: - return MACOS_VERSION_MOJAVE; + case 18: return MACOS_VERSION_MOJAVE; + case 19: return MACOS_VERSION_CATALINA; default: if (major < 9) return MACOS_VERSION_UNKNOWN; return MACOS_VERSION_UNKNOWN_NEWER; @@ -913,7 +912,7 @@ char **GetArgv() { return *_NSGetArgv(); } -#if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM +#if SANITIZER_IOS // The task_vm_info struct is normally provided by the macOS SDK, but we need // fields only available in 10.12+. Declare the struct manually to be able to // build against older SDKs. @@ -944,33 +943,37 @@ struct __sanitizer_task_vm_info { #define __SANITIZER_TASK_VM_INFO_COUNT ((mach_msg_type_number_t) \ (sizeof(__sanitizer_task_vm_info) / sizeof(natural_t))) -uptr GetTaskInfoMaxAddress() { +static uptr GetTaskInfoMaxAddress() { __sanitizer_task_vm_info vm_info = {} /* zero initialize */; mach_msg_type_number_t count = __SANITIZER_TASK_VM_INFO_COUNT; int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count); - if (err == 0 && vm_info.max_address != 0) { - return vm_info.max_address - 1; - } else { - // xnu cannot provide vm address limit - return 0x200000000 - 1; - } + return err ? 0 : vm_info.max_address; } -#endif uptr GetMaxUserVirtualAddress() { -#if SANITIZER_WORDSIZE == 64 -# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM - // Get the maximum VM address static uptr max_vm = GetTaskInfoMaxAddress(); - CHECK(max_vm); - return max_vm; + if (max_vm != 0) + return max_vm - 1; + + // xnu cannot provide vm address limit +# if SANITIZER_WORDSIZE == 32 + return 0xffe00000 - 1; # else - return (1ULL << 47) - 1; // 0x00007fffffffffffUL; + return 0x200000000 - 1; # endif -#else // SANITIZER_WORDSIZE == 32 +} + +#else // !SANITIZER_IOS + +uptr GetMaxUserVirtualAddress() { +# if SANITIZER_WORDSIZE == 64 + return (1ULL << 47) - 1; // 0x00007fffffffffffUL; +# else // SANITIZER_WORDSIZE == 32 + static_assert(SANITIZER_WORDSIZE == 32, "Wrong wordsize"); return (1ULL << 32) - 1; // 0xffffffff; -#endif // SANITIZER_WORDSIZE +# endif } +#endif uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); diff --git a/lib/sanitizer_common/sanitizer_mac.h b/lib/sanitizer_common/sanitizer_mac.h index 52825f8cf819..2257883084ea 100644 --- a/lib/sanitizer_common/sanitizer_mac.h +++ b/lib/sanitizer_common/sanitizer_mac.h @@ -1,9 +1,8 @@ //===-- sanitizer_mac.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,6 +44,7 @@ enum MacosVersion { MACOS_VERSION_HIGH_SIERRA, MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4, MACOS_VERSION_MOJAVE, + MACOS_VERSION_CATALINA, MACOS_VERSION_UNKNOWN_NEWER }; diff --git a/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/lib/sanitizer_common/sanitizer_mac_libcdep.cc index 090f17c888b4..93fb5b2f96c6 100644 --- a/lib/sanitizer_common/sanitizer_mac_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_mac_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_mac_libcdep.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_malloc_mac.inc b/lib/sanitizer_common/sanitizer_malloc_mac.inc index 44c914cea9f9..3f3581eeb484 100644 --- a/lib/sanitizer_common/sanitizer_malloc_mac.inc +++ b/lib/sanitizer_common/sanitizer_malloc_mac.inc @@ -1,9 +1,8 @@ //===-- sanitizer_malloc_mac.inc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,6 +39,8 @@ struct sanitizer_malloc_introspection_t : public malloc_introspection_t { // Used to track changes to the allocator that will affect // zone enumeration. u64 allocator_enumeration_version; + uptr allocator_ptr; + uptr allocator_size; }; u64 GetMallocZoneAllocatorEnumerationVersion() { @@ -281,13 +282,48 @@ void __sanitizer_mz_free_definite_size( } #endif -kern_return_t mi_enumerator(task_t task, void *, - unsigned type_mask, vm_address_t zone_address, - memory_reader_t reader, +#ifndef COMMON_MALLOC_HAS_ZONE_ENUMERATOR +#error "COMMON_MALLOC_HAS_ZONE_ENUMERATOR must be defined" +#endif +static_assert((COMMON_MALLOC_HAS_ZONE_ENUMERATOR) == 0 || + (COMMON_MALLOC_HAS_ZONE_ENUMERATOR) == 1, + "COMMON_MALLOC_HAS_ZONE_ENUMERATOR must be 0 or 1"); + +#if COMMON_MALLOC_HAS_ZONE_ENUMERATOR +// Forward declare and expect the implementation to provided by +// includer. +kern_return_t mi_enumerator(task_t task, void *, unsigned type_mask, + vm_address_t zone_address, memory_reader_t reader, + vm_range_recorder_t recorder); +#else +// Provide stub implementation that fails. +kern_return_t mi_enumerator(task_t task, void *, unsigned type_mask, + vm_address_t zone_address, memory_reader_t reader, vm_range_recorder_t recorder) { - // Should enumerate all the pointers we have. Seems like a lot of work. + // Not supported. return KERN_FAILURE; } +#endif + +#ifndef COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT +#error "COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT must be defined" +#endif +static_assert((COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT) == 0 || + (COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT) == 1, + "COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT must be 0 or 1"); +#if COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT +// Forward declare and expect the implementation to provided by +// includer. +void mi_extra_init( + sanitizer_malloc_introspection_t *mi); +#else +void mi_extra_init( + sanitizer_malloc_introspection_t *mi) { + // Just zero initialize the fields. + mi->allocator_ptr = 0; + mi->allocator_size = 0; +} +#endif size_t mi_good_size(malloc_zone_t *zone, size_t size) { // I think it's always safe to return size, but we maybe could do better. @@ -347,6 +383,9 @@ void InitMallocZoneFields() { sanitizer_zone_introspection.allocator_enumeration_version = GetMallocZoneAllocatorEnumerationVersion(); + // Perform any sanitizer specific initialization. + mi_extra_init(&sanitizer_zone_introspection); + internal_memset(&sanitizer_zone, 0, sizeof(malloc_zone_t)); // Use version 6 for OSX >= 10.6. diff --git a/lib/sanitizer_common/sanitizer_mutex.h b/lib/sanitizer_common/sanitizer_mutex.h index a93705be6918..40a65914299e 100644 --- a/lib/sanitizer_common/sanitizer_mutex.h +++ b/lib/sanitizer_common/sanitizer_mutex.h @@ -1,9 +1,8 @@ //===-- sanitizer_mutex.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_netbsd.cc b/lib/sanitizer_common/sanitizer_netbsd.cc index 80d0855efab0..385008e4c95a 100644 --- a/lib/sanitizer_common/sanitizer_netbsd.cc +++ b/lib/sanitizer_common/sanitizer_netbsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_netbsd.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -246,10 +245,9 @@ uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { return _REAL(__clock_gettime50, clk_id, tp); } -uptr internal_ptrace(int request, int pid, void *addr, void *data) { - Printf("internal_ptrace not implemented for NetBSD"); - Die(); - return 0; +uptr internal_ptrace(int request, int pid, void *addr, int data) { + DEFINE__REAL(int, ptrace, int a, int b, void *c, int d); + return _REAL(ptrace, request, pid, addr, data); } uptr internal_waitpid(int pid, int *status, int options) { @@ -323,11 +321,16 @@ void internal_sigemptyset(__sanitizer_sigset_t *set) { (void)_REAL(__sigemptyset14, set); } -uptr intrnal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, - int *parent_tidptr, void *newtls, int *child_tidptr) { - Printf("internal_clone not implemented for NetBSD"); - Die(); - return 0; +void internal_sigdelset(__sanitizer_sigset_t *set, int signo) { + DEFINE__REAL(int, __sigdelset14, const void *a, int b); + (void)_REAL(__sigdelset14, set, signo); +} + +uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, + void *arg) { + DEFINE__REAL(int, clone, int (*a)(void *b), void *c, int d, void *e); + + return _REAL(clone, fn, child_stack, flags, arg); } } // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_openbsd.cc b/lib/sanitizer_common/sanitizer_openbsd.cc index f0d071e51f02..b6e9bdfa87e5 100644 --- a/lib/sanitizer_common/sanitizer_openbsd.cc +++ b/lib/sanitizer_common/sanitizer_openbsd.cc @@ -1,7 +1,8 @@ //===-- sanitizer_openbsd.cc ----------------------------------------------===// // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_persistent_allocator.cc b/lib/sanitizer_common/sanitizer_persistent_allocator.cc index 5fa533a70861..ddd9ae9ec64d 100644 --- a/lib/sanitizer_common/sanitizer_persistent_allocator.cc +++ b/lib/sanitizer_common/sanitizer_persistent_allocator.cc @@ -1,9 +1,8 @@ //===-- sanitizer_persistent_allocator.cc -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_persistent_allocator.h b/lib/sanitizer_common/sanitizer_persistent_allocator.h index 8e5ce06d4c3d..de4fb6ebc3cf 100644 --- a/lib/sanitizer_common/sanitizer_persistent_allocator.h +++ b/lib/sanitizer_common/sanitizer_persistent_allocator.h @@ -1,9 +1,8 @@ //===-- sanitizer_persistent_allocator.h ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_placement_new.h b/lib/sanitizer_common/sanitizer_placement_new.h index 8904d1040ea7..1ceb8b909268 100644 --- a/lib/sanitizer_common/sanitizer_placement_new.h +++ b/lib/sanitizer_common/sanitizer_placement_new.h @@ -1,9 +1,8 @@ //===-- sanitizer_placement_new.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform.h b/lib/sanitizer_common/sanitizer_platform.h index 82bb1af7746b..b45c975358db 100644 --- a/lib/sanitizer_common/sanitizer_platform.h +++ b/lib/sanitizer_common/sanitizer_platform.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -241,10 +240,21 @@ # else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) # endif +#elif defined(__sparc__) +#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52) #else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) #endif +// Whether the addresses are sign-extended from the VMA range to the word. +// The SPARC64 Linux port implements this to split the VMA space into two +// non-contiguous halves with a huge hole in the middle. +#if defined(__sparc__) && SANITIZER_WORDSIZE == 64 +#define SANITIZER_SIGN_EXTENDED_ADDRESSES 1 +#else +#define SANITIZER_SIGN_EXTENDED_ADDRESSES 0 +#endif + // The AArch64 linux port uses the canonical syscall set as mandated by // the upstream linux community for all new ports. Other ports may still // use legacy syscalls. @@ -286,10 +296,10 @@ # define MSC_PREREQ(version) 0 #endif -#if defined(__arm64__) && SANITIZER_IOS -# define SANITIZER_NON_UNIQUE_TYPEINFO 1 -#else +#if SANITIZER_MAC && !(defined(__arm64__) && SANITIZER_IOS) # define SANITIZER_NON_UNIQUE_TYPEINFO 0 +#else +# define SANITIZER_NON_UNIQUE_TYPEINFO 1 #endif // On linux, some architectures had an ABI transition from 64-bit long double diff --git a/lib/sanitizer_common/sanitizer_platform_interceptors.h b/lib/sanitizer_common/sanitizer_platform_interceptors.h index 4d146651b78d..817d24b3492f 100644 --- a/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_interceptors.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -143,6 +142,9 @@ #define SANITIZER_INTERCEPT_MEMMOVE 1 #define SANITIZER_INTERCEPT_MEMCPY 1 #define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_BCMP \ + SANITIZER_INTERCEPT_MEMCMP && \ + ((SI_POSIX && _GNU_SOURCE) || SI_NETBSD || SI_OPENBSD || SI_FREEBSD) #define SANITIZER_INTERCEPT_STRNDUP SI_POSIX #define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD #if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ @@ -281,6 +283,9 @@ #define SANITIZER_INTERCEPT_WCRTOMB \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \ SI_SOLARIS) +#define SANITIZER_INTERCEPT_WCTOMB \ + (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \ + SI_SOLARIS) #define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_REALPATH SI_POSIX #define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME \ @@ -309,6 +314,7 @@ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX #define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX +#define SANITIZER_INTERCEPT_PTHREAD_SIGMASK SI_POSIX #define SANITIZER_INTERCEPT_BACKTRACE \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX @@ -357,6 +363,7 @@ #define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD #define SANITIZER_INTERCEPT_TMPNAM SI_POSIX #define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_TTYNAME SI_POSIX #define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX #define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX #define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS @@ -407,7 +414,8 @@ #else #define SANITIZER_INTERCEPT_AEABI_MEM 0 #endif -#define SANITIZER_INTERCEPT___BZERO SI_MAC +#define SANITIZER_INTERCEPT___BZERO SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_BZERO SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_FTIME \ (!SI_FREEBSD && !SI_NETBSD && !SI_OPENBSD && SI_POSIX) #define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS @@ -479,10 +487,12 @@ #define SANITIZER_INTERCEPT_CFREE \ (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \ SI_NOT_RTEMS) +#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX #define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC && SI_NOT_RTEMS) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_OPENBSD) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_WCSCAT SI_POSIX +#define SANITIZER_INTERCEPT_WCSDUP SI_POSIX #define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA) #define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID diff --git a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc index 377a62cab6c4..034848742697 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_freebsd.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h index 588bead2ac12..7e162a5e49d7 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_freebsd.h -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_linux.cc b/lib/sanitizer_common/sanitizer_platform_limits_linux.cc index 46e3b181304e..14b0821cb819 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_linux.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_linux.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_linux.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc index b23b430d9e5b..b2fb5cb76463 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_netbsd.cc -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -122,10 +121,11 @@ #include <dev/ic/nvmeio.h> #include <dev/ir/irdaio.h> #include <dev/isa/isvio.h> -#include <dev/isa/satlinkio.h> #include <dev/isa/wtreg.h> #include <dev/iscsi/iscsi_ioctl.h> +#if 0 #include <dev/nvmm/nvmm_ioctl.h> +#endif #include <dev/ofw/openfirmio.h> #include <dev/pci/amrio.h> #include <dev/pci/mlyreg.h> @@ -639,7 +639,6 @@ unsigned struct_rf_recon_req_sz = sizeof(rf_recon_req); unsigned struct_rio_conf_sz = sizeof(rio_conf); unsigned struct_rio_interface_sz = sizeof(rio_interface); unsigned struct_rio_stats_sz = sizeof(rio_stats); -unsigned struct_satlink_id_sz = sizeof(satlink_id); unsigned struct_scan_io_sz = sizeof(scan_io); unsigned struct_scbusaccel_args_sz = sizeof(scbusaccel_args); unsigned struct_scbusiodetach_args_sz = sizeof(scbusiodetach_args); @@ -1105,9 +1104,6 @@ unsigned IOCTL_IRDA_GET_TURNAROUNDMASK = IRDA_GET_TURNAROUNDMASK; unsigned IOCTL_IRFRAMETTY_GET_DEVICE = IRFRAMETTY_GET_DEVICE; unsigned IOCTL_IRFRAMETTY_GET_DONGLE = IRFRAMETTY_GET_DONGLE; unsigned IOCTL_IRFRAMETTY_SET_DONGLE = IRFRAMETTY_SET_DONGLE; -unsigned IOCTL_SATIORESET = SATIORESET; -unsigned IOCTL_SATIOGID = SATIOGID; -unsigned IOCTL_SATIOSBUFSIZE = SATIOSBUFSIZE; unsigned IOCTL_ISV_CMD = ISV_CMD; unsigned IOCTL_WTQICMD = WTQICMD; unsigned IOCTL_ISCSI_GET_VERSION = ISCSI_GET_VERSION; diff --git a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h index 0c0c8a837b8b..add9852ec6c3 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_netbsd.h --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -803,7 +802,6 @@ extern unsigned struct_rf_recon_req_sz; extern unsigned struct_rio_conf_sz; extern unsigned struct_rio_interface_sz; extern unsigned struct_rio_stats_sz; -extern unsigned struct_satlink_id_sz; extern unsigned struct_scan_io_sz; extern unsigned struct_scbusaccel_args_sz; extern unsigned struct_scbusiodetach_args_sz; @@ -1266,9 +1264,6 @@ extern unsigned IOCTL_IRDA_GET_TURNAROUNDMASK; extern unsigned IOCTL_IRFRAMETTY_GET_DEVICE; extern unsigned IOCTL_IRFRAMETTY_GET_DONGLE; extern unsigned IOCTL_IRFRAMETTY_SET_DONGLE; -extern unsigned IOCTL_SATIORESET; -extern unsigned IOCTL_SATIOGID; -extern unsigned IOCTL_SATIOSBUFSIZE; extern unsigned IOCTL_ISV_CMD; extern unsigned IOCTL_WTQICMD; extern unsigned IOCTL_ISCSI_GET_VERSION; diff --git a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc index 7ab0ff60a24d..5a1b07fa9c5e 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_openbsd.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h index d9899913e576..6d8b062716b7 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_openbsd.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_openbsd.h -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index ecc69bcea79b..b7fa6e8f7e07 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_posix.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/sanitizer_common/sanitizer_platform_limits_posix.h index de69852d35a6..f1a4fd7d3709 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_posix.h ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc b/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc index 15b80a81ae3b..3503eb2eab34 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_solaris.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/lib/sanitizer_common/sanitizer_platform_limits_solaris.h index 08bd24cd23e0..eb5c5855b378 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_solaris.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_solaris.h @@ -1,9 +1,8 @@ //===-- sanitizer_platform_limits_solaris.h -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_posix.cc b/lib/sanitizer_common/sanitizer_posix.cc index b965fb0da1aa..bf7127443c41 100644 --- a/lib/sanitizer_common/sanitizer_posix.cc +++ b/lib/sanitizer_common/sanitizer_posix.cc @@ -1,9 +1,8 @@ //===-- sanitizer_posix.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -44,9 +43,8 @@ uptr GetMmapGranularity() { void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { size = RoundUpTo(size, GetPageSizeCached()); - uptr res = internal_mmap(nullptr, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); + uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, mem_type); int reserrno; if (UNLIKELY(internal_iserror(res, &reserrno))) ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno, raw_report); @@ -67,9 +65,8 @@ void UnmapOrDie(void *addr, uptr size) { void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { size = RoundUpTo(size, GetPageSizeCached()); - uptr res = internal_mmap(nullptr, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); + uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, mem_type); int reserrno; if (UNLIKELY(internal_iserror(res, &reserrno))) { if (reserrno == ENOMEM) @@ -104,12 +101,9 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, } void *MmapNoReserveOrDie(uptr size, const char *mem_type) { - uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap(nullptr, - RoundUpTo(size, PageSize), - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, - -1, 0); + size = RoundUpTo(size, GetPageSizeCached()); + uptr p = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, mem_type); int reserrno; if (UNLIKELY(internal_iserror(p, &reserrno))) ReportMmapFailureAndDie(size, mem_type, "allocate noreserve", reserrno); @@ -117,13 +111,12 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) { return (void *)p; } -void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem) { - uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap((void*)(fixed_addr & ~(PageSize - 1)), - RoundUpTo(size, PageSize), - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_FIXED, - -1, 0); +static void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem, + const char *name) { + size = RoundUpTo(size, GetPageSizeCached()); + fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached()); + uptr p = MmapNamed((void *)fixed_addr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, name); int reserrno; if (UNLIKELY(internal_iserror(p, &reserrno))) { if (tolerate_enomem && reserrno == ENOMEM) @@ -137,12 +130,12 @@ void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem) { return (void *)p; } -void *MmapFixedOrDie(uptr fixed_addr, uptr size) { - return MmapFixedImpl(fixed_addr, size, false /*tolerate_enomem*/); +void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) { + return MmapFixedImpl(fixed_addr, size, false /*tolerate_enomem*/, name); } -void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) { - return MmapFixedImpl(fixed_addr, size, true /*tolerate_enomem*/); +void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) { + return MmapFixedImpl(fixed_addr, size, true /*tolerate_enomem*/, name); } bool MprotectNoAccess(uptr addr, uptr size) { @@ -344,6 +337,52 @@ bool ShouldMockFailureToOpen(const char *path) { internal_strncmp(path, "/proc/", 6) == 0; } +#if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO +int GetNamedMappingFd(const char *name, uptr size, int *flags) { + if (!common_flags()->decorate_proc_maps || !name) + return -1; + char shmname[200]; + CHECK(internal_strlen(name) < sizeof(shmname) - 10); + internal_snprintf(shmname, sizeof(shmname), "/dev/shm/%zu [%s]", + internal_getpid(), name); + int fd = ReserveStandardFds( + internal_open(shmname, O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, S_IRWXU)); + CHECK_GE(fd, 0); + int res = internal_ftruncate(fd, size); + CHECK_EQ(0, res); + res = internal_unlink(shmname); + CHECK_EQ(0, res); + *flags &= ~(MAP_ANON | MAP_ANONYMOUS); + return fd; +} +#else +int GetNamedMappingFd(const char *name, uptr size, int *flags) { + return -1; +} +#endif + +#if SANITIZER_ANDROID +#define PR_SET_VMA 0x53564d41 +#define PR_SET_VMA_ANON_NAME 0 +void DecorateMapping(uptr addr, uptr size, const char *name) { + if (!common_flags()->decorate_proc_maps || !name) + return; + internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, addr, size, (uptr)name); +} +#else +void DecorateMapping(uptr addr, uptr size, const char *name) { +} +#endif + +uptr MmapNamed(void *addr, uptr length, int prot, int flags, const char *name) { + int fd = GetNamedMappingFd(name, length, &flags); + uptr res = internal_mmap(addr, length, prot, flags, fd, 0); + if (!internal_iserror(res)) + DecorateMapping(res, length, name); + return res; +} + + } // namespace __sanitizer #endif // SANITIZER_POSIX diff --git a/lib/sanitizer_common/sanitizer_posix.h b/lib/sanitizer_common/sanitizer_posix.h index 04a76445eab2..6cf5ce75b12e 100644 --- a/lib/sanitizer_common/sanitizer_posix.h +++ b/lib/sanitizer_common/sanitizer_posix.h @@ -1,9 +1,8 @@ //===-- sanitizer_posix.h -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,7 +55,11 @@ uptr internal_unlink(const char *path); uptr internal_rename(const char *oldpath, const char *newpath); uptr internal_lseek(fd_t fd, OFF_T offset, int whence); +#if SANITIZER_NETBSD +uptr internal_ptrace(int request, int pid, void *addr, int data); +#else uptr internal_ptrace(int request, int pid, void *addr, void *data); +#endif uptr internal_waitpid(int pid, int *status, int options); int internal_fork(); @@ -105,6 +108,18 @@ fd_t ReserveStandardFds(fd_t fd); bool ShouldMockFailureToOpen(const char *path); +// Create a non-file mapping with a given /proc/self/maps name. +uptr MmapNamed(void *addr, uptr length, int prot, int flags, const char *name); + +// Platforms should implement at most one of these. +// 1. Provide a pre-decorated file descriptor to use instead of an anonymous +// mapping. +int GetNamedMappingFd(const char *name, uptr size, int *flags); +// 2. Add name to an existing anonymous mapping. The caller must keep *name +// alive at least as long as the mapping exists. +void DecorateMapping(uptr addr, uptr size, const char *name); + + } // namespace __sanitizer #endif // SANITIZER_POSIX_H diff --git a/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/lib/sanitizer_common/sanitizer_posix_libcdep.cc index 3006e60d89f3..23a2f84ac887 100644 --- a/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_posix_libcdep.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -71,7 +70,7 @@ void ReleaseMemoryPagesToOS(uptr beg, uptr end) { bool NoHugePagesInRegion(uptr addr, uptr size) { #ifdef MADV_NOHUGEPAGE // May not be defined on old systems. - return madvise((void *)addr, size, MADV_NOHUGEPAGE) == 0; + return madvise((char *)addr, size, MADV_NOHUGEPAGE) == 0; #else return true; #endif // MADV_NOHUGEPAGE @@ -79,9 +78,9 @@ bool NoHugePagesInRegion(uptr addr, uptr size) { bool DontDumpShadowMemory(uptr addr, uptr length) { #if defined(MADV_DONTDUMP) - return madvise((void *)addr, length, MADV_DONTDUMP) == 0; + return madvise((char *)addr, length, MADV_DONTDUMP) == 0; #elif defined(MADV_NOCORE) - return madvise((void *)addr, length, MADV_NOCORE) == 0; + return madvise((char *)addr, length, MADV_NOCORE) == 0; #else return true; #endif // MADV_DONTDUMP @@ -117,10 +116,6 @@ bool StackSizeIsUnlimited() { return (stack_size == RLIM_INFINITY); } -uptr GetStackSizeLimitInBytes() { - return (uptr)getlim(RLIMIT_STACK); -} - void SetStackSizeLimitInBytes(uptr limit) { setlim(RLIMIT_STACK, (rlim_t)limit); CHECK(!StackSizeIsUnlimited()); @@ -308,37 +303,11 @@ void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { MemoryMappingLayout::CacheMemoryMappings(); } -#if SANITIZER_ANDROID || SANITIZER_GO -int GetNamedMappingFd(const char *name, uptr size) { - return -1; -} -#else -int GetNamedMappingFd(const char *name, uptr size) { - if (!common_flags()->decorate_proc_maps) - return -1; - char shmname[200]; - CHECK(internal_strlen(name) < sizeof(shmname) - 10); - internal_snprintf(shmname, sizeof(shmname), "%zu [%s]", internal_getpid(), - name); - int fd = shm_open(shmname, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); - CHECK_GE(fd, 0); - int res = internal_ftruncate(fd, size); - CHECK_EQ(0, res); - res = shm_unlink(shmname); - CHECK_EQ(0, res); - return fd; -} -#endif - bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { - int fd = name ? GetNamedMappingFd(name, size) : -1; - unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; - if (fd == -1) flags |= MAP_ANON; - - uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap((void *)(fixed_addr & ~(PageSize - 1)), - RoundUpTo(size, PageSize), PROT_READ | PROT_WRITE, - flags, fd, 0); + size = RoundUpTo(size, GetPageSizeCached()); + fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached()); + uptr p = MmapNamed((void *)fixed_addr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE | MAP_ANON, name); int reserrno; if (internal_iserror(p, &reserrno)) { Report("ERROR: %s failed to " @@ -351,12 +320,8 @@ bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { } uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { - // We don't pass `name` along because, when you enable `decorate_proc_maps` - // AND actually use a named mapping AND are using a sanitizer intercepting - // `open` (e.g. TSAN, ESAN), then you'll get a failure during initialization. - // TODO(flowerhack): Fix the implementation of GetNamedMappingFd to solve - // this problem. - base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size) : MmapNoAccess(size); + base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size, name) + : MmapNoAccess(size); size_ = size; name_ = name; (void)os_handle_; // unsupported @@ -365,12 +330,14 @@ uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { // Uses fixed_addr for now. // Will use offset instead once we've implemented this function for real. -uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { - return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size)); +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) { + return reinterpret_cast<uptr>( + MmapFixedOrDieOnFatalError(fixed_addr, size, name)); } -uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { - return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size)); +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size, + const char *name) { + return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size, name)); } void ReservedAddressRange::Unmap(uptr addr, uptr size) { @@ -385,12 +352,9 @@ void ReservedAddressRange::Unmap(uptr addr, uptr size) { } void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { - int fd = name ? GetNamedMappingFd(name, size) : -1; - unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; - if (fd == -1) flags |= MAP_ANON; - - return (void *)internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd, - 0); + return (void *)MmapNamed((void *)fixed_addr, size, PROT_NONE, + MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE | MAP_ANON, + name); } void *MmapNoAccess(uptr size) { diff --git a/lib/sanitizer_common/sanitizer_printf.cc b/lib/sanitizer_common/sanitizer_printf.cc index 4315f6516c03..f4e17a8f51fd 100644 --- a/lib/sanitizer_common/sanitizer_printf.cc +++ b/lib/sanitizer_common/sanitizer_printf.cc @@ -1,9 +1,8 @@ //===-- sanitizer_printf.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps.h b/lib/sanitizer_common/sanitizer_procmaps.h index acb7104f33e6..052027111ceb 100644 --- a/lib/sanitizer_common/sanitizer_procmaps.h +++ b/lib/sanitizer_common/sanitizer_procmaps.h @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_bsd.cc b/lib/sanitizer_common/sanitizer_procmaps_bsd.cc index 362a424d737a..c38bafd9f52f 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_bsd.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_bsd.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_bsd.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_common.cc b/lib/sanitizer_common/sanitizer_procmaps_common.cc index 17d61b6c5e8f..4a2b0a047ad6 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_common.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/lib/sanitizer_common/sanitizer_procmaps_linux.cc index cf9cb25ba061..fd5e03619bdd 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_linux.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 267c960b55d4..148910f42061 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_procmaps_mac.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/lib/sanitizer_common/sanitizer_procmaps_solaris.cc index 49bb46c31b08..b5df6fe4c41e 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_solaris.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_solaris.cc @@ -1,7 +1,8 @@ //===-- sanitizer_procmaps_solaris.cc -------------------------------------===// // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,9 +50,11 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { segment->protection |= kProtectionExecute; if (segment->filename != NULL && segment->filename_size > 0) { - internal_snprintf(segment->filename, - Min(segment->filename_size, (uptr)PATH_MAX), "%s", + char proc_path[PATH_MAX + 1]; + + internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s", xmapentry->pr_mapname); + internal_readlink(proc_path, segment->filename, segment->filename_size); } data_.current += sizeof(prxmap_t); diff --git a/lib/sanitizer_common/sanitizer_quarantine.h b/lib/sanitizer_common/sanitizer_quarantine.h index d4aa12fbebce..992f23152c6a 100644 --- a/lib/sanitizer_common/sanitizer_quarantine.h +++ b/lib/sanitizer_common/sanitizer_quarantine.h @@ -1,9 +1,8 @@ //===-- sanitizer_quarantine.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_report_decorator.h b/lib/sanitizer_common/sanitizer_report_decorator.h index a7878684a60d..d276c2cdd84d 100644 --- a/lib/sanitizer_common/sanitizer_report_decorator.h +++ b/lib/sanitizer_common/sanitizer_report_decorator.h @@ -1,9 +1,8 @@ //===-- sanitizer_report_decorator.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_ring_buffer.h b/lib/sanitizer_common/sanitizer_ring_buffer.h index d15f27fd4a83..2a46e933b75d 100644 --- a/lib/sanitizer_common/sanitizer_ring_buffer.h +++ b/lib/sanitizer_common/sanitizer_ring_buffer.h @@ -1,9 +1,8 @@ //===-- sanitizer_ring_buffer.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -140,7 +139,7 @@ class CompactRingBuffer { SetNext(next); } - T operator[](uptr Idx) const { + const T &operator[](uptr Idx) const { CHECK_LT(Idx, size()); const T *Begin = (const T *)StartOfStorage(); sptr StorageIdx = Next() - Begin; diff --git a/lib/sanitizer_common/sanitizer_rtems.cc b/lib/sanitizer_common/sanitizer_rtems.cc index 76aebe419034..ffc21b958b8e 100644 --- a/lib/sanitizer_common/sanitizer_rtems.cc +++ b/lib/sanitizer_common/sanitizer_rtems.cc @@ -1,9 +1,8 @@ //===-- sanitizer_rtems.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_rtems.h b/lib/sanitizer_common/sanitizer_rtems.h index 968fa66e1be7..e8adfd500dff 100644 --- a/lib/sanitizer_common/sanitizer_rtems.h +++ b/lib/sanitizer_common/sanitizer_rtems.h @@ -1,9 +1,8 @@ //===-- sanitizer_rtems.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_signal_interceptors.inc b/lib/sanitizer_common/sanitizer_signal_interceptors.inc index f51fc049e327..68d9eb65968d 100644 --- a/lib/sanitizer_common/sanitizer_signal_interceptors.inc +++ b/lib/sanitizer_common/sanitizer_signal_interceptors.inc @@ -1,9 +1,8 @@ //===-- sanitizer_signal_interceptors.inc -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_solaris.cc b/lib/sanitizer_common/sanitizer_solaris.cc index cc0201c7a37e..37b50c6b243a 100644 --- a/lib/sanitizer_common/sanitizer_solaris.cc +++ b/lib/sanitizer_common/sanitizer_solaris.cc @@ -1,7 +1,8 @@ //===-- sanitizer_solaris.cc ----------------------------------------------===// // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -87,11 +88,6 @@ uptr internal_open(const char *filename, int flags, u32 mode) { return _REAL64(open)(filename, flags, mode); } -uptr OpenFile(const char *filename, bool write) { - return ReserveStandardFds( - internal_open(filename, write ? O_WRONLY | O_CREAT : O_RDONLY, 0660)); -} - DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) { return _REAL(read)(fd, buf, count); } @@ -124,6 +120,10 @@ uptr internal_filesize(fd_t fd) { return (uptr)st.st_size; } +DECLARE__REAL_AND_INTERNAL(uptr, dup, int oldfd) { + return _REAL(dup)(oldfd); +} + DECLARE__REAL_AND_INTERNAL(uptr, dup2, int oldfd, int newfd) { return _REAL(dup2)(oldfd, newfd); } diff --git a/lib/sanitizer_common/sanitizer_stackdepot.cc b/lib/sanitizer_common/sanitizer_stackdepot.cc index 6aab98485226..1cdedfa32fbe 100644 --- a/lib/sanitizer_common/sanitizer_stackdepot.cc +++ b/lib/sanitizer_common/sanitizer_stackdepot.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stackdepot.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "sanitizer_stackdepot.h" #include "sanitizer_common.h" +#include "sanitizer_hash.h" #include "sanitizer_stackdepotbase.h" namespace __sanitizer { @@ -50,23 +50,9 @@ struct StackDepotNode { return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr); } static u32 hash(const args_type &args) { - // murmur2 - const u32 m = 0x5bd1e995; - const u32 seed = 0x9747b28c; - const u32 r = 24; - u32 h = seed ^ (args.size * sizeof(uptr)); - for (uptr i = 0; i < args.size; i++) { - u32 k = args.trace[i]; - k *= m; - k ^= k >> r; - k *= m; - h *= m; - h ^= k; - } - h ^= h >> 13; - h *= m; - h ^= h >> 15; - return h; + MurMur2HashBuilder H(args.size * sizeof(uptr)); + for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]); + return H.get(); } static bool is_valid(const args_type &args) { return args.size > 0 && args.trace; diff --git a/lib/sanitizer_common/sanitizer_stackdepot.h b/lib/sanitizer_common/sanitizer_stackdepot.h index e22ed2e38e57..bf29cb9a006e 100644 --- a/lib/sanitizer_common/sanitizer_stackdepot.h +++ b/lib/sanitizer_common/sanitizer_stackdepot.h @@ -1,9 +1,8 @@ //===-- sanitizer_stackdepot.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stackdepotbase.h b/lib/sanitizer_common/sanitizer_stackdepotbase.h index 4ec77b467b1f..ef1b4f7f7055 100644 --- a/lib/sanitizer_common/sanitizer_stackdepotbase.h +++ b/lib/sanitizer_common/sanitizer_stackdepotbase.h @@ -1,9 +1,8 @@ //===-- sanitizer_stackdepotbase.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stacktrace.cc b/lib/sanitizer_common/sanitizer_stacktrace.cc index 21976b6b1531..26474037ff35 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,10 +17,9 @@ namespace __sanitizer { uptr StackTrace::GetNextInstructionPc(uptr pc) { -#if defined(__mips__) +#if defined(__sparc__) || defined(__mips__) return pc + 8; -#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ - defined(__aarch64__) +#elif defined(__powerpc__) || defined(__arm__) || defined(__aarch64__) return pc + 4; #else return pc + 1; @@ -50,6 +48,7 @@ void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) { static inline uhwptr *GetCanonicFrame(uptr bp, uptr stack_top, uptr stack_bottom) { + CHECK_GT(stack_top, stack_bottom); #ifdef __arm__ if (!IsValidFrame(bp, stack_top, stack_bottom)) return 0; uhwptr *bp_prev = (uhwptr *)bp; @@ -68,10 +67,11 @@ static inline uhwptr *GetCanonicFrame(uptr bp, #endif } -void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, - uptr stack_bottom, u32 max_depth) { - const uptr kPageSize = GetPageSizeCached(); +void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, + uptr stack_bottom, u32 max_depth) { + // TODO(yln): add arg sanity check for stack_top/stack_bottom CHECK_GE(max_depth, 2); + const uptr kPageSize = GetPageSizeCached(); trace_buffer[0] = pc; size = 1; if (stack_top < 4096) return; // Sanity check for stack top. diff --git a/lib/sanitizer_common/sanitizer_stacktrace.h b/lib/sanitizer_common/sanitizer_stacktrace.h index 450a40a9069f..f1f29e9f32ee 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/lib/sanitizer_common/sanitizer_stacktrace.h @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,9 +16,11 @@ namespace __sanitizer { +struct BufferedStackTrace; + static const u32 kStackTraceMax = 256; -#if defined(__sparc__) || (SANITIZER_LINUX && defined(__mips__)) +#if SANITIZER_LINUX && defined(__mips__) # define SANITIZER_CAN_FAST_UNWIND 0 #elif SANITIZER_WINDOWS # define SANITIZER_CAN_FAST_UNWIND 0 @@ -59,7 +60,7 @@ struct StackTrace { static bool WillUseFastUnwind(bool request_fast_unwind) { if (!SANITIZER_CAN_FAST_UNWIND) return false; - else if (!SANITIZER_CAN_SLOW_UNWIND) + if (!SANITIZER_CAN_SLOW_UNWIND) return true; return request_fast_unwind; } @@ -97,6 +98,23 @@ struct BufferedStackTrace : public StackTrace { BufferedStackTrace() : StackTrace(trace_buffer, 0), top_frame_bp(0) {} void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0); + + // Get the stack trace with the given pc and bp. + // The pc will be in the position 0 of the resulting stack trace. + // The bp may refer to the current frame or to the caller's frame. + void Unwind(uptr pc, uptr bp, void *context, bool request_fast, + u32 max_depth = kStackTraceMax) { + top_frame_bp = (max_depth > 0) ? bp : 0; + // Small max_depth optimization + if (max_depth <= 1) { + if (max_depth == 1) + trace_buffer[0] = pc; + size = max_depth; + return; + } + UnwindImpl(pc, bp, context, request_fast, max_depth); + } + void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind); @@ -106,16 +124,23 @@ struct BufferedStackTrace : public StackTrace { } private: - void FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, - u32 max_depth); - void SlowUnwindStack(uptr pc, u32 max_depth); - void SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth); + // Every runtime defines its own implementation of this method + void UnwindImpl(uptr pc, uptr bp, void *context, bool request_fast, + u32 max_depth); + + // UnwindFast/Slow have platform-specific implementations + void UnwindFast(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, + u32 max_depth); + void UnwindSlow(uptr pc, u32 max_depth); + void UnwindSlow(uptr pc, void *context, u32 max_depth); + void PopStackFrames(uptr count); uptr LocatePcInTrace(uptr pc); BufferedStackTrace(const BufferedStackTrace &) = delete; void operator=(const BufferedStackTrace &) = delete; + + friend class FastUnwindTest; }; // Check if given pointer points into allocated stack area. @@ -127,21 +152,23 @@ static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) { // Use this macro if you want to print stack trace with the caller // of the current function in the top frame. -#define GET_CALLER_PC_BP_SP \ - uptr bp = GET_CURRENT_FRAME(); \ - uptr pc = GET_CALLER_PC(); \ - uptr local_stack; \ - uptr sp = (uptr)&local_stack - #define GET_CALLER_PC_BP \ uptr bp = GET_CURRENT_FRAME(); \ uptr pc = GET_CALLER_PC(); +#define GET_CALLER_PC_BP_SP \ + GET_CALLER_PC_BP; \ + uptr local_stack; \ + uptr sp = (uptr)&local_stack + // Use this macro if you want to print stack trace with the current // function in the top frame. -#define GET_CURRENT_PC_BP_SP \ +#define GET_CURRENT_PC_BP \ uptr bp = GET_CURRENT_FRAME(); \ - uptr pc = StackTrace::GetCurrentPc(); \ + uptr pc = StackTrace::GetCurrentPc() + +#define GET_CURRENT_PC_BP_SP \ + GET_CURRENT_PC_BP; \ uptr local_stack; \ uptr sp = (uptr)&local_stack diff --git a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc index c87b18e1b696..859032ba84e1 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace_libcdep.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -58,6 +57,8 @@ void StackTrace::Print() const { void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind) { + // Ensures all call sites get what they requested. + CHECK_EQ(request_fast_unwind, WillUseFastUnwind(request_fast_unwind)); top_frame_bp = (max_depth > 0) ? bp : 0; // Avoid doing any work for small max_depth. if (max_depth == 0) { @@ -72,14 +73,14 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, if (!WillUseFastUnwind(request_fast_unwind)) { #if SANITIZER_CAN_SLOW_UNWIND if (context) - SlowUnwindStackWithContext(pc, context, max_depth); + UnwindSlow(pc, context, max_depth); else - SlowUnwindStack(pc, max_depth); + UnwindSlow(pc, max_depth); #else UNREACHABLE("slow unwind requested but not available"); #endif } else { - FastUnwindStack(pc, bp, stack_top, stack_bottom, max_depth); + UnwindFast(pc, bp, stack_top, stack_bottom, max_depth); } } diff --git a/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/lib/sanitizer_common/sanitizer_stacktrace_printer.cc index f2b337433427..17bbf6c0b868 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -1,9 +1,8 @@ //===-- sanitizer_common.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stacktrace_printer.h b/lib/sanitizer_common/sanitizer_stacktrace_printer.h index ce85bd7f23fa..f7f7629f773f 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/lib/sanitizer_common/sanitizer_stacktrace_printer.h @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace_printer.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc b/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc index f41a3cefb350..b238cfbc26a3 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stacktrace_sparc.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,47 +12,74 @@ // Implemention of fast stack unwinding for Sparc. //===----------------------------------------------------------------------===// -// This file is ported to Sparc v8, but it should be easy to port to -// Sparc v9. -#if defined(__sparcv8__) || defined(__sparcv8) || defined(__sparc_v8__) +#if defined(__sparc__) + +#if defined(__arch64__) || defined(__sparcv9) +#define STACK_BIAS 2047 +#else +#define STACK_BIAS 0 +#endif #include "sanitizer_common.h" #include "sanitizer_stacktrace.h" namespace __sanitizer { -void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, - uptr stack_bottom, u32 max_depth) { - const uptr kPageSize = GetPageSizeCached(); +void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, + uptr stack_bottom, u32 max_depth) { + // TODO(yln): add arg sanity check for stack_top/stack_bottom CHECK_GE(max_depth, 2); + const uptr kPageSize = GetPageSizeCached(); +#if defined(__GNUC__) + // __builtin_return_address returns the address of the call instruction + // on the SPARC and not the return address, so we need to compensate. + trace_buffer[0] = GetNextInstructionPc(pc); +#else trace_buffer[0] = pc; +#endif size = 1; if (stack_top < 4096) return; // Sanity check for stack top. // Flush register windows to memory +#if defined(__sparc_v9__) || defined(__sparcv9__) || defined(__sparcv9) + asm volatile("flushw" ::: "memory"); +#else asm volatile("ta 3" ::: "memory"); - uhwptr *frame = (uhwptr*)bp; +#endif + // On the SPARC, the return address is not in the frame, it is in a + // register. There is no way to access it off of the current frame + // pointer, but it can be accessed off the previous frame pointer by + // reading the value from the register window save area. + uptr prev_bp = GET_CURRENT_FRAME(); + uptr next_bp = prev_bp; + unsigned int i = 0; + while (next_bp != bp && IsAligned(next_bp, sizeof(uhwptr)) && i++ < 8) { + prev_bp = next_bp; + next_bp = (uptr)((uhwptr *)next_bp)[14] + STACK_BIAS; + } + if (next_bp == bp) + bp = prev_bp; // Lowest possible address that makes sense as the next frame pointer. // Goes up as we walk the stack. uptr bottom = stack_bottom; // Avoid infinite loop when frame == frame[0] by using frame > prev_frame. - while (IsValidFrame((uptr)frame, stack_top, bottom) && - IsAligned((uptr)frame, sizeof(*frame)) && + while (IsValidFrame(bp, stack_top, bottom) && IsAligned(bp, sizeof(uhwptr)) && size < max_depth) { - uhwptr pc1 = frame[15]; + uhwptr pc1 = ((uhwptr *)bp)[15]; // Let's assume that any pointer in the 0th page is invalid and // stop unwinding here. If we're adding support for a platform // where this isn't true, we need to reconsider this check. if (pc1 < kPageSize) break; if (pc1 != pc) { - trace_buffer[size++] = (uptr) pc1; + // %o7 contains the address of the call instruction and not the + // return address, so we need to compensate. + trace_buffer[size++] = GetNextInstructionPc((uptr)pc1); } - bottom = (uptr)frame; - frame = (uhwptr*)frame[14]; + bottom = bp; + bp = (uptr)((uhwptr *)bp)[14] + STACK_BIAS; } } } // namespace __sanitizer -#endif // !defined(__sparcv8__) && !defined(__sparcv8) && - // !defined(__sparc_v8__) +#endif // !defined(__sparc__) diff --git a/lib/sanitizer_common/sanitizer_stoptheworld.h b/lib/sanitizer_common/sanitizer_stoptheworld.h index 20b49ae78b85..4e4240057142 100644 --- a/lib/sanitizer_common/sanitizer_stoptheworld.h +++ b/lib/sanitizer_common/sanitizer_stoptheworld.h @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc index fe4fe86daedf..716f0d226954 100644 --- a/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld_linux_libcdep.cc ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc b/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc index d84ebef3b7f9..e79edc40f98f 100644 --- a/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc +++ b/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_stoptheworld_mac.cc -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cc b/lib/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cc new file mode 100644 index 000000000000..7a8d14ae741e --- /dev/null +++ b/lib/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cc @@ -0,0 +1,356 @@ +//===-- sanitizer_stoptheworld_netbsd_libcdep.cc --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// See sanitizer_stoptheworld.h for details. +// This implementation was inspired by Markus Gutschke's linuxthreads.cc. +// +// This is a NetBSD variation of Linux stoptheworld implementation +// See sanitizer_stoptheworld_linux_libcdep.cc for code comments. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_NETBSD + +#include "sanitizer_stoptheworld.h" + +#include "sanitizer_atomic.h" +#include "sanitizer_platform_limits_posix.h" + +#include <sys/types.h> + +#include <sys/ptrace.h> +#include <sys/uio.h> +#include <sys/wait.h> + +#include <machine/reg.h> + +#include <elf.h> +#include <errno.h> +#include <sched.h> +#include <signal.h> +#include <stddef.h> + +#define internal_sigaction_norestorer internal_sigaction + +#include "sanitizer_common.h" +#include "sanitizer_flags.h" +#include "sanitizer_libc.h" +#include "sanitizer_linux.h" +#include "sanitizer_mutex.h" +#include "sanitizer_placement_new.h" + +namespace __sanitizer { + +class SuspendedThreadsListNetBSD : public SuspendedThreadsList { + public: + SuspendedThreadsListNetBSD() { thread_ids_.reserve(1024); } + + tid_t GetThreadID(uptr index) const; + uptr ThreadCount() const; + bool ContainsTid(tid_t thread_id) const; + void Append(tid_t tid); + + PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer, + uptr *sp) const; + uptr RegisterCount() const; + + private: + InternalMmapVector<tid_t> thread_ids_; +}; + +struct TracerThreadArgument { + StopTheWorldCallback callback; + void *callback_argument; + BlockingMutex mutex; + atomic_uintptr_t done; + uptr parent_pid; +}; + +class ThreadSuspender { + public: + explicit ThreadSuspender(pid_t pid, TracerThreadArgument *arg) + : arg(arg), pid_(pid) { + CHECK_GE(pid, 0); + } + bool SuspendAllThreads(); + void ResumeAllThreads(); + void KillAllThreads(); + SuspendedThreadsListNetBSD &suspended_threads_list() { + return suspended_threads_list_; + } + TracerThreadArgument *arg; + + private: + SuspendedThreadsListNetBSD suspended_threads_list_; + pid_t pid_; +}; + +void ThreadSuspender::ResumeAllThreads() { + int pterrno; + if (!internal_iserror(internal_ptrace(PT_DETACH, pid_, (void *)(uptr)1, 0), + &pterrno)) { + VReport(2, "Detached from process %d.\n", pid_); + } else { + VReport(1, "Could not detach from process %d (errno %d).\n", pid_, pterrno); + } +} + +void ThreadSuspender::KillAllThreads() { + internal_ptrace(PT_KILL, pid_, nullptr, 0); +} + +bool ThreadSuspender::SuspendAllThreads() { + int pterrno; + if (internal_iserror(internal_ptrace(PT_ATTACH, pid_, nullptr, 0), + &pterrno)) { + Printf("Could not attach to process %d (errno %d).\n", pid_, pterrno); + return false; + } + + int status; + uptr waitpid_status; + HANDLE_EINTR(waitpid_status, internal_waitpid(pid_, &status, 0)); + + VReport(2, "Attached to process %d.\n", pid_); + + struct ptrace_lwpinfo pl; + int val; + pl.pl_lwpid = 0; + while ((val = ptrace(PT_LWPINFO, pid_, (void *)&pl, sizeof(pl))) != -1 && + pl.pl_lwpid != 0) { + suspended_threads_list_.Append(pl.pl_lwpid); + VReport(2, "Appended thread %d in process %d.\n", pl.pl_lwpid, pid_); + } + return true; +} + +// Pointer to the ThreadSuspender instance for use in signal handler. +static ThreadSuspender *thread_suspender_instance = nullptr; + +// Synchronous signals that should not be blocked. +static const int kSyncSignals[] = {SIGABRT, SIGILL, SIGFPE, SIGSEGV, + SIGBUS, SIGXCPU, SIGXFSZ}; + +static void TracerThreadDieCallback() { + ThreadSuspender *inst = thread_suspender_instance; + if (inst && stoptheworld_tracer_pid == internal_getpid()) { + inst->KillAllThreads(); + thread_suspender_instance = nullptr; + } +} + +// Signal handler to wake up suspended threads when the tracer thread dies. +static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo, + void *uctx) { + SignalContext ctx(siginfo, uctx); + Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum, + ctx.addr, ctx.pc, ctx.sp); + ThreadSuspender *inst = thread_suspender_instance; + if (inst) { + if (signum == SIGABRT) + inst->KillAllThreads(); + else + inst->ResumeAllThreads(); + RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback)); + thread_suspender_instance = nullptr; + atomic_store(&inst->arg->done, 1, memory_order_relaxed); + } + internal__exit((signum == SIGABRT) ? 1 : 2); +} + +// Size of alternative stack for signal handlers in the tracer thread. +static const int kHandlerStackSize = 8192; + +// This function will be run as a cloned task. +static int TracerThread(void *argument) { + TracerThreadArgument *tracer_thread_argument = + (TracerThreadArgument *)argument; + + // Check if parent is already dead. + if (internal_getppid() != tracer_thread_argument->parent_pid) + internal__exit(4); + + // Wait for the parent thread to finish preparations. + tracer_thread_argument->mutex.Lock(); + tracer_thread_argument->mutex.Unlock(); + + RAW_CHECK(AddDieCallback(TracerThreadDieCallback)); + + ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument); + // Global pointer for the signal handler. + thread_suspender_instance = &thread_suspender; + + // Alternate stack for signal handling. + InternalMmapVector<char> handler_stack_memory(kHandlerStackSize); + stack_t handler_stack; + internal_memset(&handler_stack, 0, sizeof(handler_stack)); + handler_stack.ss_sp = handler_stack_memory.data(); + handler_stack.ss_size = kHandlerStackSize; + internal_sigaltstack(&handler_stack, nullptr); + + // Install our handler for synchronous signals. Other signals should be + // blocked by the mask we inherited from the parent thread. + for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) { + __sanitizer_sigaction act; + internal_memset(&act, 0, sizeof(act)); + act.sigaction = TracerThreadSignalHandler; + act.sa_flags = SA_ONSTACK | SA_SIGINFO; + internal_sigaction_norestorer(kSyncSignals[i], &act, 0); + } + + int exit_code = 0; + if (!thread_suspender.SuspendAllThreads()) { + VReport(1, "Failed suspending threads.\n"); + exit_code = 3; + } else { + tracer_thread_argument->callback(thread_suspender.suspended_threads_list(), + tracer_thread_argument->callback_argument); + thread_suspender.ResumeAllThreads(); + exit_code = 0; + } + RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback)); + thread_suspender_instance = nullptr; + atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed); + return exit_code; +} + +class ScopedStackSpaceWithGuard { + public: + explicit ScopedStackSpaceWithGuard(uptr stack_size) { + stack_size_ = stack_size; + guard_size_ = GetPageSizeCached(); + // FIXME: Omitting MAP_STACK here works in current kernels but might break + // in the future. + guard_start_ = + (uptr)MmapOrDie(stack_size_ + guard_size_, "ScopedStackWithGuard"); + CHECK(MprotectNoAccess((uptr)guard_start_, guard_size_)); + } + ~ScopedStackSpaceWithGuard() { + UnmapOrDie((void *)guard_start_, stack_size_ + guard_size_); + } + void *Bottom() const { + return (void *)(guard_start_ + stack_size_ + guard_size_); + } + + private: + uptr stack_size_; + uptr guard_size_; + uptr guard_start_; +}; + +static __sanitizer_sigset_t blocked_sigset; +static __sanitizer_sigset_t old_sigset; + +struct ScopedSetTracerPID { + explicit ScopedSetTracerPID(uptr tracer_pid) { + stoptheworld_tracer_pid = tracer_pid; + stoptheworld_tracer_ppid = internal_getpid(); + } + ~ScopedSetTracerPID() { + stoptheworld_tracer_pid = 0; + stoptheworld_tracer_ppid = 0; + } +}; + +void StopTheWorld(StopTheWorldCallback callback, void *argument) { + // Prepare the arguments for TracerThread. + struct TracerThreadArgument tracer_thread_argument; + tracer_thread_argument.callback = callback; + tracer_thread_argument.callback_argument = argument; + tracer_thread_argument.parent_pid = internal_getpid(); + atomic_store(&tracer_thread_argument.done, 0, memory_order_relaxed); + const uptr kTracerStackSize = 2 * 1024 * 1024; + ScopedStackSpaceWithGuard tracer_stack(kTracerStackSize); + + tracer_thread_argument.mutex.Lock(); + + internal_sigfillset(&blocked_sigset); + for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) + internal_sigdelset(&blocked_sigset, kSyncSignals[i]); + int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset); + CHECK_EQ(rv, 0); + uptr tracer_pid = internal_clone(TracerThread, tracer_stack.Bottom(), + CLONE_VM | CLONE_FS | CLONE_FILES, + &tracer_thread_argument); + internal_sigprocmask(SIG_SETMASK, &old_sigset, 0); + int local_errno = 0; + if (internal_iserror(tracer_pid, &local_errno)) { + VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno); + tracer_thread_argument.mutex.Unlock(); + } else { + ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid); + + tracer_thread_argument.mutex.Unlock(); + + while (atomic_load(&tracer_thread_argument.done, memory_order_relaxed) == 0) + sched_yield(); + + for (;;) { + uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL); + if (!internal_iserror(waitpid_status, &local_errno)) + break; + if (local_errno == EINTR) + continue; + VReport(1, "Waiting on the tracer thread failed (errno %d).\n", + local_errno); + break; + } + } +} + +tid_t SuspendedThreadsListNetBSD::GetThreadID(uptr index) const { + CHECK_LT(index, thread_ids_.size()); + return thread_ids_[index]; +} + +uptr SuspendedThreadsListNetBSD::ThreadCount() const { + return thread_ids_.size(); +} + +bool SuspendedThreadsListNetBSD::ContainsTid(tid_t thread_id) const { + for (uptr i = 0; i < thread_ids_.size(); i++) { + if (thread_ids_[i] == thread_id) + return true; + } + return false; +} + +void SuspendedThreadsListNetBSD::Append(tid_t tid) { + thread_ids_.push_back(tid); +} + +PtraceRegistersStatus SuspendedThreadsListNetBSD::GetRegistersAndSP( + uptr index, uptr *buffer, uptr *sp) const { + lwpid_t tid = GetThreadID(index); + pid_t ppid = internal_getppid(); + struct reg regs; + int pterrno; + bool isErr = + internal_iserror(internal_ptrace(PT_GETREGS, ppid, ®s, tid), &pterrno); + if (isErr) { + VReport(1, + "Could not get registers from process %d thread %d (errno %d).\n", + ppid, tid, pterrno); + return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL + : REGISTERS_UNAVAILABLE; + } + + *sp = PTRACE_REG_SP(®s); + internal_memcpy(buffer, ®s, sizeof(regs)); + + return REGISTERS_AVAILABLE; +} + +uptr SuspendedThreadsListNetBSD::RegisterCount() const { + return sizeof(struct reg) / sizeof(uptr); +} +} // namespace __sanitizer + +#endif diff --git a/lib/sanitizer_common/sanitizer_suppressions.cc b/lib/sanitizer_common/sanitizer_suppressions.cc index 232171ed5b67..12ecd9a2e368 100644 --- a/lib/sanitizer_common/sanitizer_suppressions.cc +++ b/lib/sanitizer_common/sanitizer_suppressions.cc @@ -1,9 +1,8 @@ //===-- sanitizer_suppressions.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -31,6 +30,7 @@ SuppressionContext::SuppressionContext(const char *suppression_types[], internal_memset(has_suppression_type_, 0, suppression_types_num_); } +#if !SANITIZER_FUCHSIA static bool GetPathAssumingFileIsRelativeToExec(const char *file_path, /*out*/char *new_file_path, uptr new_file_path_size) { @@ -47,20 +47,30 @@ static bool GetPathAssumingFileIsRelativeToExec(const char *file_path, return false; } +static const char *FindFile(const char *file_path, + /*out*/char *new_file_path, + uptr new_file_path_size) { + // If we cannot find the file, check if its location is relative to + // the location of the executable. + if (!FileExists(file_path) && !IsAbsolutePath(file_path) && + GetPathAssumingFileIsRelativeToExec(file_path, new_file_path, + new_file_path_size)) { + return new_file_path; + } + return file_path; +} +#else +static const char *FindFile(const char *file_path, char *, uptr) { + return file_path; +} +#endif + void SuppressionContext::ParseFromFile(const char *filename) { if (filename[0] == '\0') return; -#if !SANITIZER_FUCHSIA - // If we cannot find the file, check if its location is relative to - // the location of the executable. InternalScopedString new_file_path(kMaxPathLength); - if (!FileExists(filename) && !IsAbsolutePath(filename) && - GetPathAssumingFileIsRelativeToExec(filename, new_file_path.data(), - new_file_path.size())) { - filename = new_file_path.data(); - } -#endif // !SANITIZER_FUCHSIA + filename = FindFile(filename, new_file_path.data(), new_file_path.size()); // Read the file. VPrintf(1, "%s: reading suppressions file at %s\n", @@ -94,7 +104,7 @@ bool SuppressionContext::Match(const char *str, const char *type, } static const char *StripPrefix(const char *str, const char *prefix) { - while (str && *str == *prefix) { + while (*str && *str == *prefix) { str++; prefix++; } diff --git a/lib/sanitizer_common/sanitizer_suppressions.h b/lib/sanitizer_common/sanitizer_suppressions.h index 0ca875a2dde6..f9da7af7e6ab 100644 --- a/lib/sanitizer_common/sanitizer_suppressions.h +++ b/lib/sanitizer_common/sanitizer_suppressions.h @@ -1,9 +1,8 @@ //===-- sanitizer_suppressions.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer.cc b/lib/sanitizer_common/sanitizer_symbolizer.cc index 672c936684be..216ce585a0c6 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -67,6 +66,16 @@ void DataInfo::Clear() { internal_memset(this, 0, sizeof(DataInfo)); } +void FrameInfo::Clear() { + InternalFree(module); + for (LocalInfo &local : locals) { + InternalFree(local.function_name); + InternalFree(local.name); + InternalFree(local.decl_file); + } + locals.clear(); +} + Symbolizer *Symbolizer::symbolizer_; StaticSpinMutex Symbolizer::init_mu_; LowLevelAllocator Symbolizer::symbolizer_allocator_; diff --git a/lib/sanitizer_common/sanitizer_symbolizer.h b/lib/sanitizer_common/sanitizer_symbolizer.h index e08eb0d660d5..51648e2d0e8d 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer.h +++ b/lib/sanitizer_common/sanitizer_symbolizer.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,6 +20,7 @@ #include "sanitizer_common.h" #include "sanitizer_mutex.h" +#include "sanitizer_vector.h" namespace __sanitizer { @@ -78,6 +78,32 @@ struct DataInfo { void Clear(); }; +struct LocalInfo { + char *function_name = nullptr; + char *name = nullptr; + char *decl_file = nullptr; + unsigned decl_line = 0; + + bool has_frame_offset = false; + bool has_size = false; + bool has_tag_offset = false; + + sptr frame_offset; + uptr size; + uptr tag_offset; + + void Clear(); +}; + +struct FrameInfo { + char *module; + uptr module_offset; + ModuleArch module_arch; + + InternalMmapVector<LocalInfo> locals; + void Clear(); +}; + class SymbolizerTool; class Symbolizer final { @@ -90,6 +116,7 @@ class Symbolizer final { // all inlined functions, if necessary). SymbolizedStack *SymbolizePC(uptr address); bool SymbolizeData(uptr address, DataInfo *info); + bool SymbolizeFrame(uptr address, FrameInfo *info); // The module names Symbolizer returns are stable and unique for every given // module. It is safe to store and compare them as pointers. diff --git a/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h b/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h index b241b9dbc3c8..c4061e38c6a4 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_fuchsia.h -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/lib/sanitizer_common/sanitizer_symbolizer_internal.h index a2ac1188241d..4611b7dfe1e5 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_internal.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_internal.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #include "sanitizer_symbolizer.h" #include "sanitizer_file.h" +#include "sanitizer_vector.h" namespace __sanitizer { @@ -59,6 +59,10 @@ class SymbolizerTool { UNIMPLEMENTED(); } + virtual bool SymbolizeFrame(uptr addr, FrameInfo *info) { + return false; + } + virtual void Flush() {} // Return nullptr to fallback to the default platform-specific demangler. @@ -121,12 +125,13 @@ class LLVMSymbolizer : public SymbolizerTool { explicit LLVMSymbolizer(const char *path, LowLevelAllocator *allocator); bool SymbolizePC(uptr addr, SymbolizedStack *stack) override; - bool SymbolizeData(uptr addr, DataInfo *info) override; + bool SymbolizeFrame(uptr addr, FrameInfo *info) override; private: - const char *FormatAndSendCommand(bool is_data, const char *module_name, - uptr module_offset, ModuleArch arch); + const char *FormatAndSendCommand(const char *command_prefix, + const char *module_name, uptr module_offset, + ModuleArch arch); LLVMSymbolizerProcess *symbolizer_process_; static const uptr kBufferSize = 16 * 1024; diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc index e98fab01d6bb..8a20e062cf47 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_libbacktrace.cc ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h index ddfd475592cb..e2a0f71420f0 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_libbacktrace.h ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc index f2ee7baad837..7f5bc55bd865 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_libcdep.cc -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -59,6 +58,16 @@ const char *ExtractUptr(const char *str, const char *delims, uptr *result) { return ret; } +const char *ExtractSptr(const char *str, const char *delims, sptr *result) { + char *buff; + const char *ret = ExtractToken(str, delims, &buff); + if (buff != 0) { + *result = (sptr)internal_atoll(buff); + } + InternalFree(buff); + return ret; +} + const char *ExtractTokenUpToDelimiter(const char *str, const char *delimiter, char **result) { const char *found_delimiter = internal_strstr(str, delimiter); @@ -113,6 +122,22 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) { return true; } +bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { + BlockingMutexLock l(&mu_); + const char *module_name; + if (!FindModuleNameAndOffsetForAddress( + addr, &module_name, &info->module_offset, &info->module_arch)) + return false; + info->module = internal_strdup(module_name); + for (auto &tool : tools_) { + SymbolizerScope sym_scope(this); + if (tool.SymbolizeFrame(addr, info)) { + return true; + } + } + return true; +} + bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, uptr *module_address) { BlockingMutexLock l(&mu_); @@ -344,10 +369,38 @@ void ParseSymbolizeDataOutput(const char *str, DataInfo *info) { str = ExtractUptr(str, "\n", &info->size); } +static void ParseSymbolizeFrameOutput(const char *str, + InternalMmapVector<LocalInfo> *locals) { + if (internal_strncmp(str, "??", 2) == 0) + return; + + while (*str) { + LocalInfo local; + str = ExtractToken(str, "\n", &local.function_name); + str = ExtractToken(str, "\n", &local.name); + + AddressInfo addr; + str = ParseFileLineInfo(&addr, str); + local.decl_file = addr.file; + local.decl_line = addr.line; + + local.has_frame_offset = internal_strncmp(str, "??", 2) != 0; + str = ExtractSptr(str, " ", &local.frame_offset); + + local.has_size = internal_strncmp(str, "??", 2) != 0; + str = ExtractUptr(str, " ", &local.size); + + local.has_tag_offset = internal_strncmp(str, "??", 2) != 0; + str = ExtractUptr(str, "\n", &local.tag_offset); + + locals->push_back(local); + } +} + bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { AddressInfo *info = &stack->info; const char *buf = FormatAndSendCommand( - /*is_data*/ false, info->module, info->module_offset, info->module_arch); + "CODE", info->module, info->module_offset, info->module_arch); if (buf) { ParseSymbolizePCOutput(buf, stack); return true; @@ -357,7 +410,7 @@ bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { const char *buf = FormatAndSendCommand( - /*is_data*/ true, info->module, info->module_offset, info->module_arch); + "DATA", info->module, info->module_offset, info->module_arch); if (buf) { ParseSymbolizeDataOutput(buf, info); info->start += (addr - info->module_offset); // Add the base address. @@ -366,22 +419,31 @@ bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { return false; } -const char *LLVMSymbolizer::FormatAndSendCommand(bool is_data, +bool LLVMSymbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { + const char *buf = FormatAndSendCommand( + "FRAME", info->module, info->module_offset, info->module_arch); + if (buf) { + ParseSymbolizeFrameOutput(buf, &info->locals); + return true; + } + return false; +} + +const char *LLVMSymbolizer::FormatAndSendCommand(const char *command_prefix, const char *module_name, uptr module_offset, ModuleArch arch) { CHECK(module_name); - const char *is_data_str = is_data ? "DATA " : ""; if (arch == kModuleArchUnknown) { - if (internal_snprintf(buffer_, kBufferSize, "%s\"%s\" 0x%zx\n", is_data_str, - module_name, + if (internal_snprintf(buffer_, kBufferSize, "%s \"%s\" 0x%zx\n", + command_prefix, module_name, module_offset) >= static_cast<int>(kBufferSize)) { Report("WARNING: Command buffer too small"); return nullptr; } } else { - if (internal_snprintf(buffer_, kBufferSize, "%s\"%s:%s\" 0x%zx\n", - is_data_str, module_name, ModuleArchToString(arch), + if (internal_snprintf(buffer_, kBufferSize, "%s \"%s:%s\" 0x%zx\n", + command_prefix, module_name, ModuleArchToString(arch), module_offset) >= static_cast<int>(kBufferSize)) { Report("WARNING: Command buffer too small"); return nullptr; diff --git a/lib/sanitizer_common/sanitizer_symbolizer_mac.cc b/lib/sanitizer_common/sanitizer_symbolizer_mac.cc index f08cb9f97cf9..d3566571948e 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_mac.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_mac.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_mac.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_mac.h b/lib/sanitizer_common/sanitizer_symbolizer_mac.h index 068644de3762..68521375e64c 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_mac.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_mac.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_mac.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_markup.cc b/lib/sanitizer_common/sanitizer_symbolizer_markup.cc index c62dc90fabba..aee49b4c4567 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_markup.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_markup.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_markup.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -118,7 +117,7 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { : _URC_NO_REASON); } -void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); size = 0; UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)}; @@ -133,9 +132,9 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { trace_buffer[0] = pc; } -void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth) { - CHECK_NE(context, nullptr); +void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { + CHECK(context); + CHECK_GE(max_depth, 2); UNREACHABLE("signal context doesn't exist"); } #endif // SANITIZER_CAN_SLOW_UNWIND diff --git a/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc index d74a6f44ee6b..2df3a90bafaa 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_posix_libcdep.cc -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_report.cc b/lib/sanitizer_common/sanitizer_symbolizer_report.cc index fd26d4cfa079..f4167d160ae8 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_report.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_report.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_report.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -104,9 +103,11 @@ void ReportMmapWriteExec(int prot) { GET_CALLER_PC_BP_SP; (void)sp; bool fast = common_flags()->fast_unwind_on_fatal; - if (fast) + if (StackTrace::WillUseFastUnwind(fast)) { GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, fast); + stack->Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, true); + } else + stack->Unwind(kStackTraceMax, pc, 0, nullptr, 0, 0, false); Printf("%s", d.Warning()); Report("WARNING: %s: writable-executable page usage\n", SanitizerToolName); diff --git a/lib/sanitizer_common/sanitizer_symbolizer_rtems.h b/lib/sanitizer_common/sanitizer_symbolizer_rtems.h index 62356ef6ef73..3371092e0687 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_rtems.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_rtems.h @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_rtems.h -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_symbolizer_win.cc b/lib/sanitizer_common/sanitizer_symbolizer_win.cc index 6ec75831a633..1badcf5f0a35 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_win.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_win.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolizer_win.cc ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,6 +35,9 @@ namespace { class WinSymbolizerTool : public SymbolizerTool { public: + // The constructor is provided to avoid synthesized memsets. + WinSymbolizerTool() {} + bool SymbolizePC(uptr addr, SymbolizedStack *stack) override; bool SymbolizeData(uptr addr, DataInfo *info) override { return false; diff --git a/lib/sanitizer_common/sanitizer_syscall_generic.inc b/lib/sanitizer_common/sanitizer_syscall_generic.inc index ddfb26397f44..a43ce3efab12 100644 --- a/lib/sanitizer_common/sanitizer_syscall_generic.inc +++ b/lib/sanitizer_common/sanitizer_syscall_generic.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_generic.inc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc index 7ab1d7641449..56c5e99220cc 100644 --- a/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc +++ b/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_linux_aarch64.inc --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc b/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc index b4fd0962a9fd..121a9445b41c 100644 --- a/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc +++ b/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_linux_arm.inc -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc index 9853a6a675d3..67e8686d12ca 100644 --- a/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc +++ b/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscall_linux_x86_64.inc ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc b/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc index 75aea2760d2c..21b521669630 100644 --- a/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc +++ b/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc @@ -1,9 +1,8 @@ //===-- sanitizer_syscalls_netbsd.inc ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_termination.cc b/lib/sanitizer_common/sanitizer_termination.cc index 35e4403adfb4..a011cce42aa2 100644 --- a/lib/sanitizer_common/sanitizer_termination.cc +++ b/lib/sanitizer_common/sanitizer_termination.cc @@ -1,9 +1,8 @@ //===-- sanitizer_termination.cc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/sanitizer_common/sanitizer_thread_registry.cc b/lib/sanitizer_common/sanitizer_thread_registry.cc index eb35cb6c5830..02691287d763 100644 --- a/lib/sanitizer_common/sanitizer_thread_registry.cc +++ b/lib/sanitizer_common/sanitizer_thread_registry.cc @@ -1,9 +1,8 @@ //===-- sanitizer_thread_registry.cc --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,8 @@ namespace __sanitizer { ThreadContextBase::ThreadContextBase(u32 tid) : tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0), - status(ThreadStatusInvalid), - detached(false), workerthread(false), parent_tid(0), next(0) { + status(ThreadStatusInvalid), detached(false), + thread_type(ThreadType::Regular), parent_tid(0), next(0) { name[0] = '\0'; atomic_store(&thread_destroyed, 0, memory_order_release); } @@ -71,11 +70,11 @@ void ThreadContextBase::SetFinished() { OnFinished(); } -void ThreadContextBase::SetStarted(tid_t _os_id, bool _workerthread, +void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg) { status = ThreadStatusRunning; os_id = _os_id; - workerthread = _workerthread; + thread_type = _thread_type; OnStarted(arg); } @@ -303,7 +302,7 @@ void ThreadRegistry::FinishThread(u32 tid) { tctx->SetDestroyed(); } -void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread, +void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg) { BlockingMutexLock l(&mtx_); running_threads_++; @@ -311,7 +310,7 @@ void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread, ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); CHECK_EQ(ThreadStatusCreated, tctx->status); - tctx->SetStarted(os_id, workerthread, arg); + tctx->SetStarted(os_id, thread_type, arg); } void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) { diff --git a/lib/sanitizer_common/sanitizer_thread_registry.h b/lib/sanitizer_common/sanitizer_thread_registry.h index 30dc603febd4..493aa988f7e6 100644 --- a/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/lib/sanitizer_common/sanitizer_thread_registry.h @@ -1,9 +1,8 @@ //===-- sanitizer_thread_registry.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,6 +28,12 @@ enum ThreadStatus { ThreadStatusDead // Joined, but some info is still available. }; +enum class ThreadType { + Regular, // Normal thread + Worker, // macOS Grand Central Dispatch (GCD) worker thread + Fiber, // Fiber +}; + // Generic thread context. Specific sanitizer tools may inherit from it. // If thread is dead, context may optionally be reused for a new thread. class ThreadContextBase { @@ -45,7 +50,7 @@ class ThreadContextBase { ThreadStatus status; bool detached; - bool workerthread; + ThreadType thread_type; u32 parent_tid; ThreadContextBase *next; // For storing thread contexts in a list. @@ -57,7 +62,7 @@ class ThreadContextBase { void SetDead(); void SetJoined(void *arg); void SetFinished(); - void SetStarted(tid_t _os_id, bool _workerthread, void *arg); + void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg); void SetCreated(uptr _user_id, u64 _unique_id, bool _detached, u32 _parent_tid, void *arg); void Reset(); @@ -121,7 +126,7 @@ class ThreadRegistry { void DetachThread(u32 tid, void *arg); void JoinThread(u32 tid, void *arg); void FinishThread(u32 tid); - void StartThread(u32 tid, tid_t os_id, bool workerthread, void *arg); + void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg); void SetThreadUserId(u32 tid, uptr user_id); private: diff --git a/lib/sanitizer_common/sanitizer_tls_get_addr.cc b/lib/sanitizer_common/sanitizer_tls_get_addr.cc index 85dc806df284..5e33e2a514f4 100644 --- a/lib/sanitizer_common/sanitizer_tls_get_addr.cc +++ b/lib/sanitizer_common/sanitizer_tls_get_addr.cc @@ -1,9 +1,8 @@ //===-- sanitizer_tls_get_addr.cc -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_tls_get_addr.h b/lib/sanitizer_common/sanitizer_tls_get_addr.h index 199a3b2e9c61..cc178d3d15ae 100644 --- a/lib/sanitizer_common/sanitizer_tls_get_addr.h +++ b/lib/sanitizer_common/sanitizer_tls_get_addr.h @@ -1,9 +1,8 @@ //===-- sanitizer_tls_get_addr.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_type_traits.cc b/lib/sanitizer_common/sanitizer_type_traits.cc index 27fec6e1f494..e3e431a13d44 100644 --- a/lib/sanitizer_common/sanitizer_type_traits.cc +++ b/lib/sanitizer_common/sanitizer_type_traits.cc @@ -1,9 +1,8 @@ //===-- sanitizer_type_traits.cc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_type_traits.h b/lib/sanitizer_common/sanitizer_type_traits.h index 4495f2c34b71..2a58d9874d2c 100644 --- a/lib/sanitizer_common/sanitizer_type_traits.h +++ b/lib/sanitizer_common/sanitizer_type_traits.h @@ -1,9 +1,8 @@ //===-- sanitizer_type_traits.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,6 +38,25 @@ struct is_same : public false_type {}; template <typename T> struct is_same<T, T> : public true_type {}; +// conditional<B, T, F> +// +// Defines type as T if B is true or as F otherwise. +// E.g. the following is true +// +// ``` +// is_same<int, conditional<true, int, double>::type>::value +// is_same<double, conditional<false, int, double>::type>::value +// ``` +template <bool B, class T, class F> +struct conditional { + using type = T; +}; + +template <class T, class F> +struct conditional<false, T, F> { + using type = F; +}; + } // namespace __sanitizer #endif diff --git a/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc index c7a5ec86fc7c..30581e848594 100644 --- a/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc @@ -1,9 +1,8 @@ //===-- sanitizer_unwind_linux_libcdep.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,7 @@ namespace __sanitizer { -//------------------------- SlowUnwindStack ----------------------------------- +//---------------------------- UnwindSlow -------------------------------------- typedef struct { uptr absolute_pc; @@ -120,7 +119,7 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { return UNWIND_CONTINUE; } -void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); size = 0; UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)}; @@ -136,14 +135,20 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { if (to_pop == 0 && size > 1) to_pop = 1; PopStackFrames(to_pop); +#if defined(__GNUC__) && defined(__sparc__) + // __builtin_return_address returns the address of the call instruction + // on the SPARC and not the return address, so we need to compensate. + trace_buffer[0] = GetNextInstructionPc(pc); +#else trace_buffer[0] = pc; +#endif } -void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { + CHECK(context); CHECK_GE(max_depth, 2); if (!unwind_backtrace_signal_arch) { - SlowUnwindStack(pc, max_depth); + UnwindSlow(pc, max_depth); return; } diff --git a/lib/sanitizer_common/sanitizer_unwind_win.cc b/lib/sanitizer_common/sanitizer_unwind_win.cc index 62bac4e9b4a5..93908ababe25 100644 --- a/lib/sanitizer_common/sanitizer_unwind_win.cc +++ b/lib/sanitizer_common/sanitizer_unwind_win.cc @@ -1,9 +1,8 @@ //===-- sanitizer_unwind_win.cc -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,7 @@ using namespace __sanitizer; #if !SANITIZER_GO -void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); // FIXME: CaptureStackBackTrace might be too slow for us. // FIXME: Compare with StackWalk64. @@ -40,8 +39,9 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { PopStackFrames(pc_location); } -void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - u32 max_depth) { +void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { + CHECK(context); + CHECK_GE(max_depth, 2); CONTEXT ctx = *(CONTEXT *)context; STACKFRAME64 stack_frame; memset(&stack_frame, 0, sizeof(stack_frame)); diff --git a/lib/sanitizer_common/sanitizer_vector.h b/lib/sanitizer_common/sanitizer_vector.h index 0632ccce262e..4b9ae7db4c13 100644 --- a/lib/sanitizer_common/sanitizer_vector.h +++ b/lib/sanitizer_common/sanitizer_vector.h @@ -1,9 +1,8 @@ //===-- sanitizer_vector.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc index 9a574dd23de6..457cecb8cec1 100644 --- a/lib/sanitizer_common/sanitizer_win.cc +++ b/lib/sanitizer_common/sanitizer_win.cc @@ -1,9 +1,8 @@ //===-- sanitizer_win.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,6 +31,18 @@ #if defined(PSAPI_VERSION) && PSAPI_VERSION == 1 #pragma comment(lib, "psapi") #endif +#if SANITIZER_WIN_TRACE +#include <traceloggingprovider.h> +// Windows trace logging provider init +#pragma comment(lib, "advapi32.lib") +TRACELOGGING_DECLARE_PROVIDER(g_asan_provider); +// GUID must be the same in utils/AddressSanitizerLoggingProvider.wprp +TRACELOGGING_DEFINE_PROVIDER(g_asan_provider, "AddressSanitizerLoggingProvider", + (0x6c6c766d, 0x3846, 0x4e6a, 0xa4, 0xfb, 0x5b, + 0x53, 0x0b, 0xd0, 0xf3, 0xfa)); +#else +#define TraceLoggingUnregister(x) +#endif // A macro to tell the compiler that this part of the code cannot be reached, // if the compiler supports this feature. Since we're using this in @@ -230,7 +241,7 @@ bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { // Memory space mapped by 'MmapFixedOrDie' must have been reserved by // 'MmapFixedNoAccess'. -void *MmapFixedOrDie(uptr fixed_addr, uptr size) { +void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) { void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_COMMIT, PAGE_READWRITE); if (p == 0) { @@ -244,11 +255,12 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) { // Uses fixed_addr for now. // Will use offset instead once we've implemented this function for real. -uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) { return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size)); } -uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size, + const char *name) { return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size)); } @@ -261,7 +273,7 @@ void ReservedAddressRange::Unmap(uptr addr, uptr size) { UnmapOrDie(reinterpret_cast<void*>(addr), size); } -void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) { +void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) { void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_COMMIT, PAGE_READWRITE); if (p == 0) { @@ -487,8 +499,14 @@ bool IsPathSeparator(const char c) { return c == '\\' || c == '/'; } +static bool IsAlpha(char c) { + c = ToLower(c); + return c >= 'a' && c <= 'z'; +} + bool IsAbsolutePath(const char *path) { - UNIMPLEMENTED(); + return path != nullptr && IsAlpha(path[0]) && path[1] == ':' && + IsPathSeparator(path[2]); } void SleepForSeconds(int seconds) { @@ -646,6 +664,7 @@ int Atexit(void (*function)(void)) { } static int RunAtexit() { + TraceLoggingUnregister(g_asan_provider); int ret = 0; for (uptr i = 0; i < atexit_functions.size(); ++i) { ret |= atexit(atexit_functions[i]); @@ -743,6 +762,7 @@ uptr internal_sched_yield() { } void internal__exit(int exitcode) { + TraceLoggingUnregister(g_asan_provider); // ExitProcess runs some finalizers, so use TerminateProcess to avoid that. // The debugger doesn't stop on TerminateProcess like it does on ExitProcess, // so add our own breakpoint here. @@ -1064,6 +1084,32 @@ u32 GetNumberOfCPUs() { return sysinfo.dwNumberOfProcessors; } +#if SANITIZER_WIN_TRACE +// TODO(mcgov): Rename this project-wide to PlatformLogInit +void AndroidLogInit(void) { + HRESULT hr = TraceLoggingRegister(g_asan_provider); + if (!SUCCEEDED(hr)) + return; +} + +void SetAbortMessage(const char *) {} + +void LogFullErrorReport(const char *buffer) { + if (common_flags()->log_to_syslog) { + InternalMmapVector<wchar_t> filename; + DWORD filename_length = 0; + do { + filename.resize(filename.size() + 0x100); + filename_length = + GetModuleFileNameW(NULL, filename.begin(), filename.size()); + } while (filename_length >= filename.size()); + TraceLoggingWrite(g_asan_provider, "AsanReportEvent", + TraceLoggingValue(filename.begin(), "ExecutableName"), + TraceLoggingValue(buffer, "AsanReportContents")); + } +} +#endif // SANITIZER_WIN_TRACE + } // namespace __sanitizer #endif // _WIN32 diff --git a/lib/sanitizer_common/sanitizer_win.h b/lib/sanitizer_common/sanitizer_win.h index 23e01ab75049..ff8939ca5e85 100644 --- a/lib/sanitizer_common/sanitizer_win.h +++ b/lib/sanitizer_common/sanitizer_win.h @@ -1,9 +1,8 @@ //===-- sanitizer_win.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_win_defs.h b/lib/sanitizer_common/sanitizer_win_defs.h index 10fc2d021ee0..bcd94a08dc44 100644 --- a/lib/sanitizer_common/sanitizer_win_defs.h +++ b/lib/sanitizer_common/sanitizer_win_defs.h @@ -1,9 +1,8 @@ //===-- sanitizer_win_defs.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/sanitizer_win_dll_thunk.cc b/lib/sanitizer_common/sanitizer_win_dll_thunk.cc index 4fb4650be478..5a947916de0a 100644 --- a/lib/sanitizer_common/sanitizer_win_dll_thunk.cc +++ b/lib/sanitizer_common/sanitizer_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- sanitizer_win_dll_thunk.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file defines a family of thunks that should be statically linked into diff --git a/lib/sanitizer_common/sanitizer_win_dll_thunk.h b/lib/sanitizer_common/sanitizer_win_dll_thunk.h index 2f9ebdaa6e76..48c73c4c98ad 100644 --- a/lib/sanitizer_common/sanitizer_win_dll_thunk.h +++ b/lib/sanitizer_common/sanitizer_win_dll_thunk.h @@ -1,9 +1,8 @@ //===-- sanitizer_win_dll_thunk.h -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This header provide helper macros to delegate calls to the shared runtime diff --git a/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc b/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc index f8f916473bf2..89756f1b70af 100644 --- a/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc +++ b/lib/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- santizer_win_dynamic_runtime_thunk.cc -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,3 +18,9 @@ #define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name) #include "sanitizer_common_interface.inc" #endif // SANITIZER_DYNAMIC_RUNTIME_THUNK + +namespace __sanitizer { +// Add one, otherwise unused, external symbol to this object file so that the +// Visual C++ linker includes it and reads the .drective section. +void ForceWholeArchiveIncludeForSanitizerCommon() {} +} diff --git a/lib/sanitizer_common/sanitizer_win_weak_interception.cc b/lib/sanitizer_common/sanitizer_win_weak_interception.cc index 5711f5dc8b6a..b1ac44d75ceb 100644 --- a/lib/sanitizer_common/sanitizer_win_weak_interception.cc +++ b/lib/sanitizer_common/sanitizer_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- sanitizer_win_weak_interception.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in the sanitizer when it is implemented as a diff --git a/lib/sanitizer_common/sanitizer_win_weak_interception.h b/lib/sanitizer_common/sanitizer_win_weak_interception.h index 5b122971d2d0..5e4d8b8def3e 100644 --- a/lib/sanitizer_common/sanitizer_win_weak_interception.h +++ b/lib/sanitizer_common/sanitizer_win_weak_interception.h @@ -1,9 +1,8 @@ //===-- sanitizer_win_weak_interception.h ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This header provide helper macros to delegate calls of weak functions to the diff --git a/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc b/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc index b25a53d73cb5..8150b7a09a0e 100644 --- a/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc +++ b/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cc @@ -1,9 +1,8 @@ //===-- sanitizer_symbolize.cc ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,8 +37,11 @@ bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset, { llvm::raw_string_ostream OS(Result); llvm::symbolize::DIPrinter Printer(OS); - auto ResOrErr = - getDefaultSymbolizer()->symbolizeInlinedCode(ModuleName, ModuleOffset); + // TODO: it is neccessary to set proper SectionIndex here. + // object::SectionedAddress::UndefSection works for only absolute addresses. + auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode( + ModuleName, + {ModuleOffset, llvm::object::SectionedAddress::UndefSection}); Printer << (ResOrErr ? ResOrErr.get() : llvm::DIInliningInfo()); } return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s", @@ -52,8 +54,11 @@ bool __sanitizer_symbolize_data(const char *ModuleName, uint64_t ModuleOffset, { llvm::raw_string_ostream OS(Result); llvm::symbolize::DIPrinter Printer(OS); - auto ResOrErr = - getDefaultSymbolizer()->symbolizeData(ModuleName, ModuleOffset); + // TODO: it is neccessary to set proper SectionIndex here. + // object::SectionedAddress::UndefSection works for only absolute addresses. + auto ResOrErr = getDefaultSymbolizer()->symbolizeData( + ModuleName, + {ModuleOffset, llvm::object::SectionedAddress::UndefSection}); Printer << (ResOrErr ? ResOrErr.get() : llvm::DIGlobal()); } return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s", diff --git a/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc b/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc index 66d089a0e6c0..c85ebe5e2e29 100644 --- a/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc +++ b/lib/sanitizer_common/symbolizer/sanitizer_wrappers.cc @@ -1,9 +1,8 @@ //===-- sanitizer_wrappers.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh b/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh index 788cef85a581..5c77bea83294 100755 --- a/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh +++ b/lib/sanitizer_common/symbolizer/scripts/ar_to_bc.sh @@ -9,8 +9,8 @@ if [ "$#" -le 1 ]; then usage fi -AR=$(readlink -f $AR) -LINK=$(readlink -f $LINK) +[[ $AR == /* ]] || AR=$PWD/$AR +[[ $LINK == /* ]] || LINK=$PWD/$LINK INPUTS= OUTPUT= diff --git a/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt b/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt index beee0acf484e..2a0f76b80ef4 100644 --- a/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt +++ b/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt @@ -36,6 +36,7 @@ __umoddi3 U _exit U abort U access U +bcmp U calloc U catclose U catgets U @@ -128,10 +129,12 @@ strrchr U strsep U strtod_l U strtof_l U +strtok_r U strtol U strtold_l U strtoll_l U strtoull_l U +syscall U tcgetattr U uname U ungetc U diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp index fb04fb281c4b..b2ebc9705930 100644 --- a/lib/scudo/scudo_allocator.cpp +++ b/lib/scudo/scudo_allocator.cpp @@ -1,9 +1,8 @@ //===-- scudo_allocator.cpp -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -26,6 +25,12 @@ #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_quarantine.h" +#ifdef GWP_ASAN_HOOKS +# include "gwp_asan/guarded_pool_allocator.h" +# include "gwp_asan/optional/backtrace.h" +# include "gwp_asan/optional/options_parser.h" +#endif // GWP_ASAN_HOOKS + #include <errno.h> #include <string.h> @@ -214,6 +219,10 @@ QuarantineCacheT *getQuarantineCache(ScudoTSD *TSD) { return reinterpret_cast<QuarantineCacheT *>(TSD->QuarantineCachePlaceHolder); } +#ifdef GWP_ASAN_HOOKS +static gwp_asan::GuardedPoolAllocator GuardedAlloc; +#endif // GWP_ASAN_HOOKS + struct Allocator { static const uptr MaxAllowedMallocSize = FIRST_32_SECOND_64(2UL << 30, 1ULL << 40); @@ -292,6 +301,14 @@ struct Allocator { void *allocate(uptr Size, uptr Alignment, AllocType Type, bool ForceZeroContents = false) { initThreadMaybe(); + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.shouldSample())) { + if (void *Ptr = GuardedAlloc.allocate(Size)) + return Ptr; + } +#endif // GWP_ASAN_HOOKS + if (UNLIKELY(Alignment > MaxAlignment)) { if (AllocatorMayReturnNull()) return nullptr; @@ -435,6 +452,14 @@ struct Allocator { __sanitizer_free_hook(Ptr); if (UNLIKELY(!Ptr)) return; + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { + GuardedAlloc.deallocate(Ptr); + return; + } +#endif // GWP_ASAN_HOOKS + if (UNLIKELY(!Chunk::isAligned(Ptr))) dieWithMessage("misaligned pointer when deallocating address %p\n", Ptr); UnpackedHeader Header; @@ -464,6 +489,18 @@ struct Allocator { // size still fits in the chunk. void *reallocate(void *OldPtr, uptr NewSize) { initThreadMaybe(); + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) { + size_t OldSize = GuardedAlloc.getSize(OldPtr); + void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); + if (NewPtr) + memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize); + GuardedAlloc.deallocate(OldPtr); + return NewPtr; + } +#endif // GWP_ASAN_HOOKS + if (UNLIKELY(!Chunk::isAligned(OldPtr))) dieWithMessage("misaligned address when reallocating address %p\n", OldPtr); @@ -505,6 +542,12 @@ struct Allocator { initThreadMaybe(); if (UNLIKELY(!Ptr)) return 0; + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) + return GuardedAlloc.getSize(Ptr); +#endif // GWP_ASAN_HOOKS + UnpackedHeader Header; Chunk::loadHeader(Ptr, &Header); // Getting the usable size of a chunk only makes sense if it's allocated. @@ -588,11 +631,11 @@ NOINLINE void Allocator::performSanityChecks() { } // Opportunistic RSS limit check. This will update the RSS limit status, if -// it can, every 100ms, otherwise it will just return the current one. +// it can, every 250ms, otherwise it will just return the current one. NOINLINE bool Allocator::isRssLimitExceeded() { u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); const u64 CurrentCheck = MonotonicNanoTime(); - if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL))) + if (LIKELY(CurrentCheck < LastCheck + (250ULL * 1000000ULL))) return atomic_load_relaxed(&RssLimitExceeded); if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, CurrentCheck, memory_order_relaxed)) @@ -627,6 +670,13 @@ static BackendT &getBackend() { void initScudo() { Instance.init(); +#ifdef GWP_ASAN_HOOKS + gwp_asan::options::initOptions(); + gwp_asan::options::Options &Opts = gwp_asan::options::getOptions(); + Opts.Backtrace = gwp_asan::options::getBacktraceFunction(); + Opts.PrintBacktrace = gwp_asan::options::getPrintBacktraceFunction(); + GuardedAlloc.init(Opts); +#endif // GWP_ASAN_HOOKS } void ScudoTSD::init() { diff --git a/lib/scudo/scudo_allocator.h b/lib/scudo/scudo_allocator.h index 814bb08ab013..0efa5c520296 100644 --- a/lib/scudo/scudo_allocator.h +++ b/lib/scudo/scudo_allocator.h @@ -1,9 +1,8 @@ //===-- scudo_allocator.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -85,12 +84,6 @@ struct AP64 { }; typedef SizeClassAllocator64<AP64> PrimaryT; #else -static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog; -# if SANITIZER_WORDSIZE == 32 -typedef FlatByteMap<NumRegions> ByteMap; -# elif SANITIZER_WORDSIZE == 64 -typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap; -# endif // SANITIZER_WORDSIZE struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; @@ -98,7 +91,6 @@ struct AP32 { typedef __scudo::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = RegionSizeLog; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __scudo::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator32FlagMasks::kRandomShuffleChunks | @@ -108,11 +100,13 @@ typedef SizeClassAllocator32<AP32> PrimaryT; #endif // SANITIZER_CAN_USE_ALLOCATOR64 #include "scudo_allocator_secondary.h" -#include "scudo_allocator_combined.h" -typedef SizeClassAllocatorLocalCache<PrimaryT> AllocatorCacheT; typedef LargeMmapAllocator SecondaryT; -typedef CombinedAllocator<PrimaryT, AllocatorCacheT, SecondaryT> BackendT; + +#include "scudo_allocator_combined.h" + +typedef CombinedAllocator BackendT; +typedef CombinedAllocator::AllocatorCache AllocatorCacheT; void initScudo(); diff --git a/lib/scudo/scudo_allocator_combined.h b/lib/scudo/scudo_allocator_combined.h index 6e40660ba9ab..d61cc9ec1a52 100644 --- a/lib/scudo/scudo_allocator_combined.h +++ b/lib/scudo/scudo_allocator_combined.h @@ -1,9 +1,8 @@ //===-- scudo_allocator_combined.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -19,10 +18,11 @@ # error "This file must be included inside scudo_allocator.h." #endif -template <class PrimaryAllocator, class AllocatorCache, - class SecondaryAllocator> class CombinedAllocator { public: + using PrimaryAllocator = PrimaryT; + using SecondaryAllocator = SecondaryT; + using AllocatorCache = typename PrimaryAllocator::AllocatorCache; void init(s32 ReleaseToOSIntervalMs) { Primary.Init(ReleaseToOSIntervalMs); Secondary.Init(); diff --git a/lib/scudo/scudo_allocator_secondary.h b/lib/scudo/scudo_allocator_secondary.h index ff6246e25883..151ff9931f6c 100644 --- a/lib/scudo/scudo_allocator_secondary.h +++ b/lib/scudo/scudo_allocator_secondary.h @@ -1,9 +1,8 @@ //===-- scudo_allocator_secondary.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_crc32.cpp b/lib/scudo/scudo_crc32.cpp index a267dc4e3fb8..87473505fe79 100644 --- a/lib/scudo/scudo_crc32.cpp +++ b/lib/scudo/scudo_crc32.cpp @@ -1,9 +1,8 @@ //===-- scudo_crc32.cpp -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_crc32.h b/lib/scudo/scudo_crc32.h index e89e430f4085..bad15a929a3e 100644 --- a/lib/scudo/scudo_crc32.h +++ b/lib/scudo/scudo_crc32.h @@ -1,9 +1,8 @@ //===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_errors.cpp b/lib/scudo/scudo_errors.cpp index d11e03cf9163..34e57bf71016 100644 --- a/lib/scudo/scudo_errors.cpp +++ b/lib/scudo/scudo_errors.cpp @@ -1,9 +1,8 @@ //===-- scudo_errors.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_errors.h b/lib/scudo/scudo_errors.h index 8b1af996be04..258695c2c02c 100644 --- a/lib/scudo/scudo_errors.h +++ b/lib/scudo/scudo_errors.h @@ -1,9 +1,8 @@ //===-- scudo_errors.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_flags.cpp b/lib/scudo/scudo_flags.cpp index c012471a8368..c564e217b35b 100644 --- a/lib/scudo/scudo_flags.cpp +++ b/lib/scudo/scudo_flags.cpp @@ -1,9 +1,8 @@ //===-- scudo_flags.cpp -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -69,7 +68,7 @@ void initFlags() { ScudoParser.ParseString(getScudoDefaultOptions()); // Override from environment. - ScudoParser.ParseString(GetEnv("SCUDO_OPTIONS")); + ScudoParser.ParseStringFromEnv("SCUDO_OPTIONS"); InitializeCommonFlags(); diff --git a/lib/scudo/scudo_flags.h b/lib/scudo/scudo_flags.h index d4ae31031fbb..483c79621cbf 100644 --- a/lib/scudo/scudo_flags.h +++ b/lib/scudo/scudo_flags.h @@ -1,9 +1,8 @@ //===-- scudo_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_flags.inc b/lib/scudo/scudo_flags.inc index f180478fdac3..c124738c1f3a 100644 --- a/lib/scudo/scudo_flags.inc +++ b/lib/scudo/scudo_flags.inc @@ -1,9 +1,8 @@ //===-- scudo_flags.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -37,7 +36,9 @@ SCUDO_FLAG(int, QuarantineChunksUpToSize, -1, "Size in bytes up to which chunks will be quarantined (if lower than" "or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)") -SCUDO_FLAG(bool, DeallocationTypeMismatch, true, +// Disable the deallocation type check by default on Android, it causes too many +// issues with third party libraries. +SCUDO_FLAG(bool, DeallocationTypeMismatch, !SANITIZER_ANDROID, "Report errors on malloc/delete, new/free, new/delete[], etc.") SCUDO_FLAG(bool, DeleteSizeMismatch, true, diff --git a/lib/scudo/scudo_interface_internal.h b/lib/scudo/scudo_interface_internal.h index 3e520a50c83b..75c63aa6d489 100644 --- a/lib/scudo/scudo_interface_internal.h +++ b/lib/scudo/scudo_interface_internal.h @@ -1,9 +1,8 @@ //===-- scudo_interface_internal.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_malloc.cpp b/lib/scudo/scudo_malloc.cpp index eef776809bd2..a72b861e28ee 100644 --- a/lib/scudo/scudo_malloc.cpp +++ b/lib/scudo/scudo_malloc.cpp @@ -1,9 +1,8 @@ //===-- scudo_malloc.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_new_delete.cpp b/lib/scudo/scudo_new_delete.cpp index daa3b47dc727..03eef7f28bb9 100644 --- a/lib/scudo/scudo_new_delete.cpp +++ b/lib/scudo/scudo_new_delete.cpp @@ -1,9 +1,8 @@ //===-- scudo_new_delete.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_platform.h b/lib/scudo/scudo_platform.h index 3a6f4be69dbd..07d4b70fc8e9 100644 --- a/lib/scudo/scudo_platform.h +++ b/lib/scudo/scudo_platform.h @@ -1,9 +1,8 @@ //===-- scudo_platform.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -80,7 +79,7 @@ const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20; #endif // SANITIZER_CAN_USE_ALLOCATOR64 #if !defined(SCUDO_SIZE_CLASS_MAP) -# define SCUDO_SIZE_CLASS_MAP Default +# define SCUDO_SIZE_CLASS_MAP Dense #endif #define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP) diff --git a/lib/scudo/scudo_termination.cpp b/lib/scudo/scudo_termination.cpp index 4237d3bc1865..6c7c0abc6d36 100644 --- a/lib/scudo/scudo_termination.cpp +++ b/lib/scudo/scudo_termination.cpp @@ -1,9 +1,8 @@ //===-- scudo_termination.cpp -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd.h b/lib/scudo/scudo_tsd.h index 2bd78716af69..1d4e4e6f126e 100644 --- a/lib/scudo/scudo_tsd.h +++ b/lib/scudo/scudo_tsd.h @@ -1,9 +1,8 @@ //===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd_exclusive.cpp b/lib/scudo/scudo_tsd_exclusive.cpp index 74e797580be7..a203a74bbcf8 100644 --- a/lib/scudo/scudo_tsd_exclusive.cpp +++ b/lib/scudo/scudo_tsd_exclusive.cpp @@ -1,9 +1,8 @@ //===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd_exclusive.inc b/lib/scudo/scudo_tsd_exclusive.inc index 1fa9dcdfd20d..08e4d3af7316 100644 --- a/lib/scudo/scudo_tsd_exclusive.inc +++ b/lib/scudo/scudo_tsd_exclusive.inc @@ -1,9 +1,8 @@ //===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_tsd_shared.cpp b/lib/scudo/scudo_tsd_shared.cpp index 8853894c00a2..59ad2549998c 100644 --- a/lib/scudo/scudo_tsd_shared.cpp +++ b/lib/scudo/scudo_tsd_shared.cpp @@ -1,9 +1,8 @@ //===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -84,9 +83,7 @@ ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD) { } const uptr Precedence = TSDs[Index].getPrecedence(); // A 0 precedence here means another thread just locked this TSD. - if (UNLIKELY(Precedence == 0)) - continue; - if (Precedence < LowestPrecedence) { + if (Precedence && Precedence < LowestPrecedence) { CandidateTSD = &TSDs[Index]; LowestPrecedence = Precedence; } diff --git a/lib/scudo/scudo_tsd_shared.inc b/lib/scudo/scudo_tsd_shared.inc index 9dad756b5386..8f3362dd3d71 100644 --- a/lib/scudo/scudo_tsd_shared.inc +++ b/lib/scudo/scudo_tsd_shared.inc @@ -1,9 +1,8 @@ //===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_utils.cpp b/lib/scudo/scudo_utils.cpp index d5788d20ca46..5e76a4a30f10 100644 --- a/lib/scudo/scudo_utils.cpp +++ b/lib/scudo/scudo_utils.cpp @@ -1,9 +1,8 @@ //===-- scudo_utils.cpp -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/scudo_utils.h b/lib/scudo/scudo_utils.h index 43448e0831e8..a8dfbdeb3b70 100644 --- a/lib/scudo/scudo_utils.h +++ b/lib/scudo/scudo_utils.h @@ -1,9 +1,8 @@ //===-- scudo_utils.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/scudo/standalone/allocator_config.h b/lib/scudo/standalone/allocator_config.h new file mode 100644 index 000000000000..06ec4f3f795a --- /dev/null +++ b/lib/scudo/standalone/allocator_config.h @@ -0,0 +1,80 @@ +//===-- allocator_config.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ALLOCATOR_CONFIG_H_ +#define SCUDO_ALLOCATOR_CONFIG_H_ + +#include "combined.h" +#include "common.h" +#include "flags.h" +#include "primary32.h" +#include "primary64.h" +#include "size_class_map.h" +#include "tsd_exclusive.h" +#include "tsd_shared.h" + +namespace scudo { + +// Default configurations for various platforms. + +struct DefaultConfig { + using SizeClassMap = DefaultSizeClassMap; +#if SCUDO_CAN_USE_PRIMARY64 + // 1GB Regions + typedef SizeClassAllocator64<SizeClassMap, 30U> Primary; +#else + // 512KB regions + typedef SizeClassAllocator32<SizeClassMap, 19U> Primary; +#endif + template <class A> using TSDRegistryT = TSDRegistryExT<A>; // Exclusive +}; + +struct AndroidConfig { + using SizeClassMap = AndroidSizeClassMap; +#if SCUDO_CAN_USE_PRIMARY64 + // 1GB regions + typedef SizeClassAllocator64<SizeClassMap, 30U> Primary; +#else + // 512KB regions + typedef SizeClassAllocator32<SizeClassMap, 19U> Primary; +#endif + template <class A> + using TSDRegistryT = TSDRegistrySharedT<A, 2U>; // Shared, max 2 TSDs. +}; + +struct AndroidSvelteConfig { + using SizeClassMap = SvelteSizeClassMap; +#if SCUDO_CAN_USE_PRIMARY64 + // 512MB regions + typedef SizeClassAllocator64<SizeClassMap, 29U> Primary; +#else + // 256KB regions + typedef SizeClassAllocator32<SizeClassMap, 18U> Primary; +#endif + template <class A> + using TSDRegistryT = TSDRegistrySharedT<A, 1U>; // Shared, only 1 TSD. +}; + +struct FuchsiaConfig { + // 1GB Regions + typedef SizeClassAllocator64<DefaultSizeClassMap, 30U> Primary; + template <class A> + using TSDRegistryT = TSDRegistrySharedT<A, 8U>; // Shared, max 8 TSDs. +}; + +#if SCUDO_ANDROID +typedef AndroidConfig Config; +#elif SCUDO_FUCHSIA +typedef FuchsiaConfig Config; +#else +typedef DefaultConfig Config; +#endif + +} // namespace scudo + +#endif // SCUDO_ALLOCATOR_CONFIG_H_ diff --git a/lib/scudo/standalone/atomic_helpers.h b/lib/scudo/standalone/atomic_helpers.h new file mode 100644 index 000000000000..47037d764e25 --- /dev/null +++ b/lib/scudo/standalone/atomic_helpers.h @@ -0,0 +1,139 @@ +//===-- atomic_helpers.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ATOMIC_H_ +#define SCUDO_ATOMIC_H_ + +#include "internal_defs.h" + +namespace scudo { + +enum memory_order { + memory_order_relaxed = 0, + memory_order_consume = 1, + memory_order_acquire = 2, + memory_order_release = 3, + memory_order_acq_rel = 4, + memory_order_seq_cst = 5 +}; +COMPILER_CHECK(memory_order_relaxed == __ATOMIC_RELAXED); +COMPILER_CHECK(memory_order_consume == __ATOMIC_CONSUME); +COMPILER_CHECK(memory_order_acquire == __ATOMIC_ACQUIRE); +COMPILER_CHECK(memory_order_release == __ATOMIC_RELEASE); +COMPILER_CHECK(memory_order_acq_rel == __ATOMIC_ACQ_REL); +COMPILER_CHECK(memory_order_seq_cst == __ATOMIC_SEQ_CST); + +struct atomic_u8 { + typedef u8 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_u16 { + typedef u16 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_s32 { + typedef s32 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_u32 { + typedef u32 Type; + volatile Type ValDoNotUse; +}; + +struct atomic_u64 { + typedef u64 Type; + // On 32-bit platforms u64 is not necessarily aligned on 8 bytes. + ALIGNED(8) volatile Type ValDoNotUse; +}; + +struct atomic_uptr { + typedef uptr Type; + volatile Type ValDoNotUse; +}; + +template <typename T> +INLINE typename T::Type atomic_load(const volatile T *A, memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + typename T::Type V; + __atomic_load(&A->ValDoNotUse, &V, MO); + return V; +} + +template <typename T> +INLINE void atomic_store(volatile T *A, typename T::Type V, memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + __atomic_store(&A->ValDoNotUse, &V, MO); +} + +INLINE void atomic_thread_fence(memory_order) { __sync_synchronize(); } + +template <typename T> +INLINE typename T::Type atomic_fetch_add(volatile T *A, typename T::Type V, + memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + return __atomic_fetch_add(&A->ValDoNotUse, V, MO); +} + +template <typename T> +INLINE typename T::Type atomic_fetch_sub(volatile T *A, typename T::Type V, + memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + return __atomic_fetch_sub(&A->ValDoNotUse, V, MO); +} + +template <typename T> +INLINE typename T::Type atomic_exchange(volatile T *A, typename T::Type V, + memory_order MO) { + DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); + typename T::Type R; + __atomic_exchange(&A->ValDoNotUse, &V, &R, MO); + return R; +} + +template <typename T> +INLINE bool atomic_compare_exchange_strong(volatile T *A, typename T::Type *Cmp, + typename T::Type Xchg, + memory_order MO) { + return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, false, MO, + __ATOMIC_RELAXED); +} + +template <typename T> +INLINE bool atomic_compare_exchange_weak(volatile T *A, typename T::Type *Cmp, + typename T::Type Xchg, + memory_order MO) { + return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, true, MO, + __ATOMIC_RELAXED); +} + +// Clutter-reducing helpers. + +template <typename T> +INLINE typename T::Type atomic_load_relaxed(const volatile T *A) { + return atomic_load(A, memory_order_relaxed); +} + +template <typename T> +INLINE void atomic_store_relaxed(volatile T *A, typename T::Type V) { + atomic_store(A, V, memory_order_relaxed); +} + +template <typename T> +INLINE typename T::Type atomic_compare_exchange(volatile T *A, + typename T::Type Cmp, + typename T::Type Xchg) { + atomic_compare_exchange_strong(A, &Cmp, Xchg, memory_order_acquire); + return Cmp; +} + +} // namespace scudo + +#endif // SCUDO_ATOMIC_H_ diff --git a/lib/scudo/standalone/bytemap.h b/lib/scudo/standalone/bytemap.h new file mode 100644 index 000000000000..caeeb2fac879 --- /dev/null +++ b/lib/scudo/standalone/bytemap.h @@ -0,0 +1,111 @@ +//===-- bytemap.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_BYTEMAP_H_ +#define SCUDO_BYTEMAP_H_ + +#include "atomic_helpers.h" +#include "common.h" +#include "mutex.h" + +namespace scudo { + +template <uptr Size> class FlatByteMap { +public: + void initLinkerInitialized() { + Map = reinterpret_cast<u8 *>(map(nullptr, Size, "scudo:bytemap")); + } + void init() { initLinkerInitialized(); } + + void unmapTestOnly() { unmap(reinterpret_cast<void *>(Map), Size); } + + void set(uptr Index, u8 Value) { + DCHECK_LT(Index, Size); + DCHECK_EQ(0U, Map[Index]); + Map[Index] = Value; + } + u8 operator[](uptr Index) { + DCHECK_LT(Index, Size); + return Map[Index]; + } + +private: + u8 *Map; +}; + +template <uptr Level1Size, uptr Level2Size> class TwoLevelByteMap { +public: + void initLinkerInitialized() { + Level1Map = reinterpret_cast<atomic_uptr *>( + map(nullptr, sizeof(atomic_uptr) * Level1Size, "scudo:bytemap")); + } + void init() { + Mutex.init(); + initLinkerInitialized(); + } + + void reset() { + for (uptr I = 0; I < Level1Size; I++) { + u8 *P = get(I); + if (!P) + continue; + unmap(P, Level2Size); + } + memset(Level1Map, 0, sizeof(atomic_uptr) * Level1Size); + } + + void unmapTestOnly() { + reset(); + unmap(reinterpret_cast<void *>(Level1Map), + sizeof(atomic_uptr) * Level1Size); + } + + uptr size() const { return Level1Size * Level2Size; } + + void set(uptr Index, u8 Value) { + DCHECK_LT(Index, Level1Size * Level2Size); + u8 *Level2Map = getOrCreate(Index / Level2Size); + DCHECK_EQ(0U, Level2Map[Index % Level2Size]); + Level2Map[Index % Level2Size] = Value; + } + + u8 operator[](uptr Index) const { + DCHECK_LT(Index, Level1Size * Level2Size); + u8 *Level2Map = get(Index / Level2Size); + if (!Level2Map) + return 0; + return Level2Map[Index % Level2Size]; + } + +private: + u8 *get(uptr Index) const { + DCHECK_LT(Index, Level1Size); + return reinterpret_cast<u8 *>( + atomic_load(&Level1Map[Index], memory_order_acquire)); + } + + u8 *getOrCreate(uptr Index) { + u8 *Res = get(Index); + if (!Res) { + ScopedLock L(Mutex); + if (!(Res = get(Index))) { + Res = reinterpret_cast<u8 *>(map(nullptr, Level2Size, "scudo:bytemap")); + atomic_store(&Level1Map[Index], reinterpret_cast<uptr>(Res), + memory_order_release); + } + } + return Res; + } + + atomic_uptr *Level1Map; + HybridMutex Mutex; +}; + +} // namespace scudo + +#endif // SCUDO_BYTEMAP_H_ diff --git a/lib/scudo/standalone/checksum.cc b/lib/scudo/standalone/checksum.cc new file mode 100644 index 000000000000..0896d5bdccd5 --- /dev/null +++ b/lib/scudo/standalone/checksum.cc @@ -0,0 +1,70 @@ +//===-- checksum.cc ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "checksum.h" +#include "atomic_helpers.h" + +#if defined(__x86_64__) || defined(__i386__) +#include <cpuid.h> +#elif defined(__arm__) || defined(__aarch64__) +#if SCUDO_FUCHSIA +#include <zircon/features.h> +#include <zircon/syscalls.h> +#else +#include <sys/auxv.h> +#endif +#endif + +namespace scudo { + +Checksum HashAlgorithm = {Checksum::BSD}; + +#if defined(__x86_64__) || defined(__i386__) +// i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. +// CRC32 requires the SSE 4.2 instruction set. +#ifndef bit_SSE4_2 +#define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. +#endif + +bool hasHardwareCRC32() { + u32 Eax, Ebx = 0, Ecx = 0, Edx = 0; + __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx); + const bool IsIntel = (Ebx == signature_INTEL_ebx) && + (Edx == signature_INTEL_edx) && + (Ecx == signature_INTEL_ecx); + const bool IsAMD = (Ebx == signature_AMD_ebx) && (Edx == signature_AMD_edx) && + (Ecx == signature_AMD_ecx); + if (!IsIntel && !IsAMD) + return false; + __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); + return !!(Ecx & bit_SSE4_2); +} + +#elif defined(__arm__) || defined(__aarch64__) +#ifndef AT_HWCAP +#define AT_HWCAP 16 +#endif +#ifndef HWCAP_CRC32 +#define HWCAP_CRC32 (1U << 7) // HWCAP_CRC32 is missing on older platforms. +#endif + +bool hasHardwareCRC32() { +#if SCUDO_FUCHSIA + u32 HWCap; + const zx_status_t Status = + zx_system_get_features(ZX_FEATURE_KIND_CPU, &HWCap); + if (Status != ZX_OK) + return false; + return !!(HWCap & ZX_ARM64_FEATURE_ISA_CRC32); +#else + return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); +#endif // SCUDO_FUCHSIA +} +#endif // defined(__x86_64__) || defined(__i386__) + +} // namespace scudo diff --git a/lib/scudo/standalone/checksum.h b/lib/scudo/standalone/checksum.h new file mode 100644 index 000000000000..092342fd6efb --- /dev/null +++ b/lib/scudo/standalone/checksum.h @@ -0,0 +1,54 @@ +//===-- checksum.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CHECKSUM_H_ +#define SCUDO_CHECKSUM_H_ + +#include "internal_defs.h" + +// Hardware CRC32 is supported at compilation via the following: +// - for i386 & x86_64: -msse4.2 +// - for ARM & AArch64: -march=armv8-a+crc or -mcrc +// An additional check must be performed at runtime as well to make sure the +// emitted instructions are valid on the target host. + +#ifdef __SSE4_2__ +#include <smmintrin.h> +#define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) +#endif +#ifdef __ARM_FEATURE_CRC32 +#include <arm_acle.h> +#define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) +#endif + +namespace scudo { + +enum class Checksum : u8 { + BSD = 0, + HardwareCRC32 = 1, +}; + +// BSD checksum, unlike a software CRC32, doesn't use any array lookup. We save +// significantly on memory accesses, as well as 1K of CRC32 table, on platforms +// that do no support hardware CRC32. The checksum itself is 16-bit, which is at +// odds with CRC32, but enough for our needs. +INLINE u16 computeBSDChecksum(u16 Sum, uptr Data) { + for (u8 I = 0; I < sizeof(Data); I++) { + Sum = static_cast<u16>((Sum >> 1) | ((Sum & 1) << 15)); + Sum = static_cast<u16>(Sum + (Data & 0xff)); + Data >>= 8; + } + return Sum; +} + +bool hasHardwareCRC32(); +WEAK u32 computeHardwareCRC32(u32 Crc, uptr Data); + +} // namespace scudo + +#endif // SCUDO_CHECKSUM_H_ diff --git a/lib/scudo/standalone/chunk.h b/lib/scudo/standalone/chunk.h new file mode 100644 index 000000000000..76ef661b0dc5 --- /dev/null +++ b/lib/scudo/standalone/chunk.h @@ -0,0 +1,156 @@ +//===-- chunk.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CHUNK_H_ +#define SCUDO_CHUNK_H_ + +#include "platform.h" + +#include "atomic_helpers.h" +#include "checksum.h" +#include "common.h" +#include "report.h" + +namespace scudo { + +extern Checksum HashAlgorithm; + +INLINE u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) { + // If the hardware CRC32 feature is defined here, it was enabled everywhere, + // as opposed to only for crc32_hw.cc. This means that other hardware specific + // instructions were likely emitted at other places, and as a result there is + // no reason to not use it here. +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + u32 Crc = static_cast<u32>(CRC32_INTRINSIC(Seed, Value)); + for (uptr I = 0; I < ArraySize; I++) + Crc = static_cast<u32>(CRC32_INTRINSIC(Crc, Array[I])); + return static_cast<u16>((Crc & 0xffff) ^ (Crc >> 16)); +#else + if (HashAlgorithm == Checksum::HardwareCRC32) { + u32 Crc = computeHardwareCRC32(Seed, Value); + for (uptr I = 0; I < ArraySize; I++) + Crc = computeHardwareCRC32(Crc, Array[I]); + return static_cast<u16>((Crc & 0xffff) ^ (Crc >> 16)); + } else { + u16 Checksum = computeBSDChecksum(static_cast<u16>(Seed & 0xffff), Value); + for (uptr I = 0; I < ArraySize; I++) + Checksum = computeBSDChecksum(Checksum, Array[I]); + return Checksum; + } +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +} + +namespace Chunk { + +// Note that in an ideal world, `State` and `Origin` should be `enum class`, and +// the associated `UnpackedHeader` fields of their respective enum class type +// but https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414 prevents it from +// happening, as it will error, complaining the number of bits is not enough. +enum Origin : u8 { + Malloc = 0, + New = 1, + NewArray = 2, + Memalign = 3, +}; + +enum State : u8 { Available = 0, Allocated = 1, Quarantined = 2 }; + +typedef u64 PackedHeader; +// Update the 'Mask' constants to reflect changes in this structure. +struct UnpackedHeader { + u64 Checksum : 16; + u64 ClassId : 8; + u64 SizeOrUnusedBytes : 20; + u8 State : 2; + u8 Origin : 2; + u64 Offset : 16; +}; +typedef atomic_u64 AtomicPackedHeader; +COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader)); + +// Those constants are required to silence some -Werror=conversion errors when +// assigning values to the related bitfield variables. +constexpr uptr ChecksumMask = (1UL << 16) - 1; +constexpr uptr ClassIdMask = (1UL << 8) - 1; +constexpr uptr SizeOrUnusedBytesMask = (1UL << 20) - 1; +constexpr uptr StateMask = (1UL << 2) - 1; +constexpr uptr OriginMask = (1UL << 2) - 1; +constexpr uptr OffsetMask = (1UL << 16) - 1; + +constexpr uptr getHeaderSize() { + return roundUpTo(sizeof(PackedHeader), 1U << SCUDO_MIN_ALIGNMENT_LOG); +} + +INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) { + return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) - + getHeaderSize()); +} + +INLINE +const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { + return reinterpret_cast<const AtomicPackedHeader *>( + reinterpret_cast<uptr>(Ptr) - getHeaderSize()); +} + +// We do not need a cryptographically strong hash for the checksum, but a CRC +// type function that can alert us in the event a header is invalid or +// corrupted. Ideally slightly better than a simple xor of all fields. +static INLINE u16 computeHeaderChecksum(u32 Cookie, const void *Ptr, + UnpackedHeader *Header) { + UnpackedHeader ZeroChecksumHeader = *Header; + ZeroChecksumHeader.Checksum = 0; + uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; + memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); + return computeChecksum(Cookie, reinterpret_cast<uptr>(Ptr), HeaderHolder, + ARRAY_SIZE(HeaderHolder)); +} + +INLINE void storeHeader(u32 Cookie, void *Ptr, + UnpackedHeader *NewUnpackedHeader) { + NewUnpackedHeader->Checksum = + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); + PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader); + atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); +} + +INLINE +void loadHeader(u32 Cookie, const void *Ptr, + UnpackedHeader *NewUnpackedHeader) { + PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); + *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader); + if (UNLIKELY(NewUnpackedHeader->Checksum != + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader))) + reportHeaderCorruption(const_cast<void *>(Ptr)); +} + +INLINE void compareExchangeHeader(u32 Cookie, void *Ptr, + UnpackedHeader *NewUnpackedHeader, + UnpackedHeader *OldUnpackedHeader) { + NewUnpackedHeader->Checksum = + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); + PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader); + PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader); + if (UNLIKELY(!atomic_compare_exchange_strong( + getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, + memory_order_relaxed))) + reportHeaderRace(Ptr); +} + +INLINE +bool isValid(u32 Cookie, const void *Ptr, UnpackedHeader *NewUnpackedHeader) { + PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); + *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader); + return NewUnpackedHeader->Checksum == + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); +} + +} // namespace Chunk + +} // namespace scudo + +#endif // SCUDO_CHUNK_H_ diff --git a/lib/scudo/standalone/combined.h b/lib/scudo/standalone/combined.h new file mode 100644 index 000000000000..4c1c1196bf8f --- /dev/null +++ b/lib/scudo/standalone/combined.h @@ -0,0 +1,557 @@ +//===-- combined.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_COMBINED_H_ +#define SCUDO_COMBINED_H_ + +#include "chunk.h" +#include "common.h" +#include "flags.h" +#include "flags_parser.h" +#include "interface.h" +#include "local_cache.h" +#include "quarantine.h" +#include "report.h" +#include "secondary.h" +#include "tsd.h" + +namespace scudo { + +template <class Params> class Allocator { +public: + using PrimaryT = typename Params::Primary; + using CacheT = typename PrimaryT::CacheT; + typedef Allocator<Params> ThisT; + typedef typename Params::template TSDRegistryT<ThisT> TSDRegistryT; + + struct QuarantineCallback { + explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache) + : Allocator(Instance), Cache(LocalCache) {} + + // Chunk recycling function, returns a quarantined chunk to the backend, + // first making sure it hasn't been tampered with. + void recycle(void *Ptr) { + Chunk::UnpackedHeader Header; + Chunk::loadHeader(Allocator.Cookie, Ptr, &Header); + if (UNLIKELY(Header.State != Chunk::State::Quarantined)) + reportInvalidChunkState(AllocatorAction::Recycling, Ptr); + + Chunk::UnpackedHeader NewHeader = Header; + NewHeader.State = Chunk::State::Available; + Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); + + void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader); + const uptr ClassId = Header.ClassId; + if (ClassId) + Cache.deallocate(ClassId, BlockBegin); + else + Allocator.Secondary.deallocate(BlockBegin); + } + + // We take a shortcut when allocating a quarantine batch by working with the + // appropriate class ID instead of using Size. The compiler should optimize + // the class ID computation and work with the associated cache directly. + void *allocate(UNUSED uptr Size) { + const uptr QuarantineClassId = SizeClassMap::getClassIdBySize( + sizeof(QuarantineBatch) + Chunk::getHeaderSize()); + void *Ptr = Cache.allocate(QuarantineClassId); + // Quarantine batch allocation failure is fatal. + if (UNLIKELY(!Ptr)) + reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId)); + + Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) + + Chunk::getHeaderSize()); + Chunk::UnpackedHeader Header = {}; + Header.ClassId = QuarantineClassId & Chunk::ClassIdMask; + Header.SizeOrUnusedBytes = sizeof(QuarantineBatch); + Header.State = Chunk::State::Allocated; + Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); + + return Ptr; + } + + void deallocate(void *Ptr) { + const uptr QuarantineClassId = SizeClassMap::getClassIdBySize( + sizeof(QuarantineBatch) + Chunk::getHeaderSize()); + Chunk::UnpackedHeader Header; + Chunk::loadHeader(Allocator.Cookie, Ptr, &Header); + + if (UNLIKELY(Header.State != Chunk::State::Allocated)) + reportInvalidChunkState(AllocatorAction::Deallocating, Ptr); + DCHECK_EQ(Header.ClassId, QuarantineClassId); + DCHECK_EQ(Header.Offset, 0); + DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch)); + + Chunk::UnpackedHeader NewHeader = Header; + NewHeader.State = Chunk::State::Available; + Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); + Cache.deallocate(QuarantineClassId, + reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) - + Chunk::getHeaderSize())); + } + + private: + ThisT &Allocator; + CacheT &Cache; + }; + + typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT; + typedef typename QuarantineT::CacheT QuarantineCacheT; + + void initLinkerInitialized() { + performSanityChecks(); + + // Check if hardware CRC32 is supported in the binary and by the platform, + // if so, opt for the CRC32 hardware version of the checksum. + if (&computeHardwareCRC32 && hasHardwareCRC32()) + HashAlgorithm = Checksum::HardwareCRC32; + + if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie)))) + Cookie = static_cast<u32>(getMonotonicTime() ^ + (reinterpret_cast<uptr>(this) >> 4)); + + initFlags(); + reportUnrecognizedFlags(); + + // Store some flags locally. + Options.MayReturnNull = getFlags()->may_return_null; + Options.ZeroContents = getFlags()->zero_contents; + Options.DeallocTypeMismatch = getFlags()->dealloc_type_mismatch; + Options.DeleteSizeMismatch = getFlags()->delete_size_mismatch; + Options.QuarantineMaxChunkSize = getFlags()->quarantine_max_chunk_size; + + Stats.initLinkerInitialized(); + Primary.initLinkerInitialized(getFlags()->release_to_os_interval_ms); + Secondary.initLinkerInitialized(&Stats); + + Quarantine.init(getFlags()->quarantine_size_kb << 10, + getFlags()->thread_local_quarantine_size_kb << 10); + } + + void reset() { memset(this, 0, sizeof(*this)); } + + void unmapTestOnly() { + TSDRegistry.unmapTestOnly(); + Primary.unmapTestOnly(); + } + + TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } + + void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); } + + // Release the resources used by a TSD, which involves: + // - draining the local quarantine cache to the global quarantine; + // - releasing the cached pointers back to the Primary; + // - unlinking the local stats from the global ones (destroying the cache does + // the last two items). + void commitBack(TSD<ThisT> *TSD) { + Quarantine.drain(&TSD->QuarantineCache, + QuarantineCallback(*this, TSD->Cache)); + TSD->Cache.destroy(&Stats); + } + + NOINLINE void *allocate(uptr Size, Chunk::Origin Origin, + uptr Alignment = MinAlignment, + bool ZeroContents = false) { + initThreadMaybe(); + + if (UNLIKELY(Alignment > MaxAlignment)) { + if (Options.MayReturnNull) + return nullptr; + reportAlignmentTooBig(Alignment, MaxAlignment); + } + if (UNLIKELY(Alignment < MinAlignment)) + Alignment = MinAlignment; + + // If the requested size happens to be 0 (more common than you might think), + // allocate 1 byte on top of the header. Then add the extra bytes required + // to fulfill the alignment requirements: we allocate enough to be sure that + // there will be an address in the block that will satisfy the alignment. + const uptr NeededSize = + Chunk::getHeaderSize() + roundUpTo(Size ? Size : 1, MinAlignment) + + ((Alignment > MinAlignment) ? (Alignment - Chunk::getHeaderSize()) : 0); + + // Takes care of extravagantly large sizes as well as integer overflows. + if (UNLIKELY(Size >= MaxAllowedMallocSize || + NeededSize >= MaxAllowedMallocSize)) { + if (Options.MayReturnNull) + return nullptr; + reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize); + } + + void *Block; + uptr ClassId; + uptr BlockEnd = 0; + if (PrimaryT::canAllocate(NeededSize)) { + ClassId = SizeClassMap::getClassIdBySize(NeededSize); + bool UnlockRequired; + auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + Block = TSD->Cache.allocate(ClassId); + if (UnlockRequired) + TSD->unlock(); + } else { + ClassId = 0; + Block = Secondary.allocate(NeededSize, Alignment, &BlockEnd); + } + + if (UNLIKELY(!Block)) { + if (Options.MayReturnNull) + return nullptr; + reportOutOfMemory(NeededSize); + } + + // We only need to zero the contents for Primary backed allocations. + if ((ZeroContents || Options.ZeroContents) && ClassId) + memset(Block, 0, PrimaryT::getSizeByClassId(ClassId)); + + Chunk::UnpackedHeader Header = {}; + uptr UserPtr = reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(); + // The following condition isn't necessarily "UNLIKELY". + if (!isAligned(UserPtr, Alignment)) { + const uptr AlignedUserPtr = roundUpTo(UserPtr, Alignment); + const uptr Offset = AlignedUserPtr - UserPtr; + Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask; + DCHECK_GT(Offset, 2 * sizeof(u32)); + // The BlockMarker has no security purpose, but is specifically meant for + // the chunk iteration function that can be used in debugging situations. + // It is the only situation where we have to locate the start of a chunk + // based on its block address. + reinterpret_cast<u32 *>(Block)[0] = BlockMarker; + reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset); + UserPtr = AlignedUserPtr; + } + Header.State = Chunk::State::Allocated; + Header.Origin = Origin & Chunk::OriginMask; + if (ClassId) { + Header.ClassId = ClassId & Chunk::ClassIdMask; + Header.SizeOrUnusedBytes = Size & Chunk::SizeOrUnusedBytesMask; + } else { + Header.SizeOrUnusedBytes = + (BlockEnd - (UserPtr + Size)) & Chunk::SizeOrUnusedBytesMask; + } + void *Ptr = reinterpret_cast<void *>(UserPtr); + Chunk::storeHeader(Cookie, Ptr, &Header); + + if (&__scudo_allocate_hook) + __scudo_allocate_hook(Ptr, Size); + + return Ptr; + } + + NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0, + UNUSED uptr Alignment = MinAlignment) { + // For a deallocation, we only ensure minimal initialization, meaning thread + // local data will be left uninitialized for now (when using ELF TLS). The + // fallback cache will be used instead. This is a workaround for a situation + // where the only heap operation performed in a thread would be a free past + // the TLS destructors, ending up in initialized thread specific data never + // being destroyed properly. Any other heap operation will do a full init. + initThreadMaybe(/*MinimalInit=*/true); + + if (&__scudo_deallocate_hook) + __scudo_deallocate_hook(Ptr); + + if (UNLIKELY(!Ptr)) + return; + if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment))) + reportMisalignedPointer(AllocatorAction::Deallocating, Ptr); + + Chunk::UnpackedHeader Header; + Chunk::loadHeader(Cookie, Ptr, &Header); + + if (UNLIKELY(Header.State != Chunk::State::Allocated)) + reportInvalidChunkState(AllocatorAction::Deallocating, Ptr); + if (Options.DeallocTypeMismatch) { + if (Header.Origin != Origin) { + // With the exception of memalign'd chunks, that can be still be free'd. + if (UNLIKELY(Header.Origin != Chunk::Origin::Memalign || + Origin != Chunk::Origin::Malloc)) + reportDeallocTypeMismatch(AllocatorAction::Deallocating, Ptr, + Header.Origin, Origin); + } + } + + const uptr Size = getSize(Ptr, &Header); + if (DeleteSize && Options.DeleteSizeMismatch) { + if (UNLIKELY(DeleteSize != Size)) + reportDeleteSizeMismatch(Ptr, DeleteSize, Size); + } + + quarantineOrDeallocateChunk(Ptr, &Header, Size); + } + + void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) { + initThreadMaybe(); + + // The following cases are handled by the C wrappers. + DCHECK_NE(OldPtr, nullptr); + DCHECK_NE(NewSize, 0); + + if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment))) + reportMisalignedPointer(AllocatorAction::Reallocating, OldPtr); + + Chunk::UnpackedHeader OldHeader; + Chunk::loadHeader(Cookie, OldPtr, &OldHeader); + + if (UNLIKELY(OldHeader.State != Chunk::State::Allocated)) + reportInvalidChunkState(AllocatorAction::Reallocating, OldPtr); + + // Pointer has to be allocated with a malloc-type function. Some + // applications think that it is OK to realloc a memalign'ed pointer, which + // will trigger this check. It really isn't. + if (Options.DeallocTypeMismatch) { + if (UNLIKELY(OldHeader.Origin != Chunk::Origin::Malloc)) + reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr, + OldHeader.Origin, Chunk::Origin::Malloc); + } + + const uptr OldSize = getSize(OldPtr, &OldHeader); + // If the new size is identical to the old one, or lower but within an + // acceptable range, we just keep the old chunk, and update its header. + if (NewSize == OldSize) + return OldPtr; + if (NewSize < OldSize) { + const uptr Delta = OldSize - NewSize; + if (Delta < (SizeClassMap::MaxSize / 2)) { + Chunk::UnpackedHeader NewHeader = OldHeader; + NewHeader.SizeOrUnusedBytes = + (OldHeader.ClassId ? NewHeader.SizeOrUnusedBytes - Delta + : NewHeader.SizeOrUnusedBytes + Delta) & + Chunk::SizeOrUnusedBytesMask; + Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); + return OldPtr; + } + } + + // Otherwise we allocate a new one, and deallocate the old one. Some + // allocators will allocate an even larger chunk (by a fixed factor) to + // allow for potential further in-place realloc. The gains of such a trick + // are currently unclear. + void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment); + if (NewPtr) { + memcpy(NewPtr, OldPtr, Min(NewSize, OldSize)); + quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize); + } + return NewPtr; + } + + // TODO(kostyak): while this locks the Primary & Secondary, it still allows + // pointers to be fetched from the TSD. We ultimately want to + // lock the registry as well. For now, it's good enough. + void disable() { + initThreadMaybe(); + Primary.disable(); + Secondary.disable(); + } + + void enable() { + initThreadMaybe(); + Secondary.enable(); + Primary.enable(); + } + + void printStats() { + disable(); + Primary.printStats(); + Secondary.printStats(); + Quarantine.printStats(); + enable(); + } + + void releaseToOS() { Primary.releaseToOS(); } + + // Iterate over all chunks and call a callback for all busy chunks located + // within the provided memory range. Said callback must not use this allocator + // or a deadlock can ensue. This fits Android's malloc_iterate() needs. + void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback, + void *Arg) { + initThreadMaybe(); + const uptr From = Base; + const uptr To = Base + Size; + auto Lambda = [this, From, To, Callback, Arg](uptr Block) { + if (Block < From || Block > To) + return; + uptr ChunkSize; + const uptr ChunkBase = getChunkFromBlock(Block, &ChunkSize); + if (ChunkBase != InvalidChunk) + Callback(ChunkBase, ChunkSize, Arg); + }; + Primary.iterateOverBlocks(Lambda); + Secondary.iterateOverBlocks(Lambda); + } + + bool canReturnNull() { + initThreadMaybe(); + return Options.MayReturnNull; + } + + // TODO(kostyak): implement this as a "backend" to mallopt. + bool setOption(UNUSED uptr Option, UNUSED uptr Value) { return false; } + + // Return the usable size for a given chunk. Technically we lie, as we just + // report the actual size of a chunk. This is done to counteract code actively + // writing past the end of a chunk (like sqlite3) when the usable size allows + // for it, which then forces realloc to copy the usable size of a chunk as + // opposed to its actual size. + uptr getUsableSize(const void *Ptr) { + initThreadMaybe(); + if (UNLIKELY(!Ptr)) + return 0; + Chunk::UnpackedHeader Header; + Chunk::loadHeader(Cookie, Ptr, &Header); + // Getting the usable size of a chunk only makes sense if it's allocated. + if (UNLIKELY(Header.State != Chunk::State::Allocated)) + reportInvalidChunkState(AllocatorAction::Sizing, const_cast<void *>(Ptr)); + return getSize(Ptr, &Header); + } + + void getStats(StatCounters S) { + initThreadMaybe(); + Stats.get(S); + } + +private: + typedef MapAllocator SecondaryT; + typedef typename PrimaryT::SizeClassMap SizeClassMap; + + static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG; + static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable. + static const uptr MinAlignment = 1UL << MinAlignmentLog; + static const uptr MaxAlignment = 1UL << MaxAlignmentLog; + static const uptr MaxAllowedMallocSize = + FIRST_32_SECOND_64(1UL << 31, 1ULL << 40); + + // Constants used by the chunk iteration mechanism. + static const u32 BlockMarker = 0x44554353U; + static const uptr InvalidChunk = ~static_cast<uptr>(0); + + GlobalStats Stats; + TSDRegistryT TSDRegistry; + PrimaryT Primary; + SecondaryT Secondary; + QuarantineT Quarantine; + + u32 Cookie; + + struct { + u8 MayReturnNull : 1; // may_return_null + u8 ZeroContents : 1; // zero_contents + u8 DeallocTypeMismatch : 1; // dealloc_type_mismatch + u8 DeleteSizeMismatch : 1; // delete_size_mismatch + u32 QuarantineMaxChunkSize; // quarantine_max_chunk_size + } Options; + + // The following might get optimized out by the compiler. + NOINLINE void performSanityChecks() { + // Verify that the header offset field can hold the maximum offset. In the + // case of the Secondary allocator, it takes care of alignment and the + // offset will always be small. In the case of the Primary, the worst case + // scenario happens in the last size class, when the backend allocation + // would already be aligned on the requested alignment, which would happen + // to be the maximum alignment that would fit in that size class. As a + // result, the maximum offset will be at most the maximum alignment for the + // last size class minus the header size, in multiples of MinAlignment. + Chunk::UnpackedHeader Header = {}; + const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex( + SizeClassMap::MaxSize - MinAlignment); + const uptr MaxOffset = + (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog; + Header.Offset = MaxOffset & Chunk::OffsetMask; + if (UNLIKELY(Header.Offset != MaxOffset)) + reportSanityCheckError("offset"); + + // Verify that we can fit the maximum size or amount of unused bytes in the + // header. Given that the Secondary fits the allocation to a page, the worst + // case scenario happens in the Primary. It will depend on the second to + // last and last class sizes, as well as the dynamic base for the Primary. + // The following is an over-approximation that works for our needs. + const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1; + Header.SizeOrUnusedBytes = + MaxSizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask; + if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes)) + reportSanityCheckError("size (or unused bytes)"); + + const uptr LargestClassId = SizeClassMap::LargestClassId; + Header.ClassId = LargestClassId; + if (UNLIKELY(Header.ClassId != LargestClassId)) + reportSanityCheckError("class ID"); + } + + static INLINE void *getBlockBegin(const void *Ptr, + Chunk::UnpackedHeader *Header) { + return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) - + Chunk::getHeaderSize() - + (Header->Offset << MinAlignmentLog)); + } + + // Return the size of a chunk as requested during its allocation. + INLINE uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) { + const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; + if (Header->ClassId) + return SizeOrUnusedBytes; + return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) - + reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes; + } + + ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { + TSDRegistry.initThreadMaybe(this, MinimalInit); + } + + void quarantineOrDeallocateChunk(void *Ptr, Chunk::UnpackedHeader *Header, + uptr Size) { + Chunk::UnpackedHeader NewHeader = *Header; + // If the quarantine is disabled, the actual size of a chunk is 0 or larger + // than the maximum allowed, we return a chunk directly to the backend. + const bool BypassQuarantine = !Quarantine.getCacheSize() || !Size || + (Size > Options.QuarantineMaxChunkSize); + if (BypassQuarantine) { + NewHeader.State = Chunk::State::Available; + Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); + void *BlockBegin = getBlockBegin(Ptr, &NewHeader); + const uptr ClassId = NewHeader.ClassId; + if (ClassId) { + bool UnlockRequired; + auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + TSD->Cache.deallocate(ClassId, BlockBegin); + if (UnlockRequired) + TSD->unlock(); + } else { + Secondary.deallocate(BlockBegin); + } + } else { + NewHeader.State = Chunk::State::Quarantined; + Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); + bool UnlockRequired; + auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + Quarantine.put(&TSD->QuarantineCache, + QuarantineCallback(*this, TSD->Cache), Ptr, Size); + if (UnlockRequired) + TSD->unlock(); + } + } + + // This only cares about valid busy chunks. This might change in the future. + uptr getChunkFromBlock(uptr Block, uptr *Size) { + u32 Offset = 0; + if (reinterpret_cast<u32 *>(Block)[0] == BlockMarker) + Offset = reinterpret_cast<u32 *>(Block)[1]; + const uptr P = Block + Offset + Chunk::getHeaderSize(); + const void *Ptr = reinterpret_cast<const void *>(P); + Chunk::UnpackedHeader Header; + if (!Chunk::isValid(Cookie, Ptr, &Header) || + Header.State != Chunk::State::Allocated) + return InvalidChunk; + if (Size) + *Size = getSize(Ptr, &Header); + return P; + } +}; + +} // namespace scudo + +#endif // SCUDO_COMBINED_H_ diff --git a/lib/scudo/standalone/common.cc b/lib/scudo/standalone/common.cc new file mode 100644 index 000000000000..2a26efbb9c89 --- /dev/null +++ b/lib/scudo/standalone/common.cc @@ -0,0 +1,32 @@ +//===-- common.cc -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.h" +#include "atomic_helpers.h" + +namespace scudo { + +uptr PageSizeCached; +uptr getPageSize(); + +uptr getPageSizeSlow() { + PageSizeCached = getPageSize(); + CHECK_NE(PageSizeCached, 0); + return PageSizeCached; +} + +// Fatal internal map() or unmap() error (potentially OOM related). +void NORETURN dieOnMapUnmapError(bool OutOfMemory) { + outputRaw("Scudo ERROR: internal map or unmap failure"); + if (OutOfMemory) + outputRaw(" (OOM)"); + outputRaw("\n"); + die(); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/common.h b/lib/scudo/standalone/common.h new file mode 100644 index 000000000000..c015d1ca5669 --- /dev/null +++ b/lib/scudo/standalone/common.h @@ -0,0 +1,176 @@ +//===-- common.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_COMMON_H_ +#define SCUDO_COMMON_H_ + +#include "internal_defs.h" + +#include "fuchsia.h" +#include "linux.h" + +#include <stddef.h> +#include <string.h> + +namespace scudo { + +template <class Dest, class Source> INLINE Dest bit_cast(const Source &S) { + COMPILER_CHECK(sizeof(Dest) == sizeof(Source)); + Dest D; + memcpy(&D, &S, sizeof(D)); + return D; +} + +INLINE constexpr uptr roundUpTo(uptr X, uptr Boundary) { + return (X + Boundary - 1) & ~(Boundary - 1); +} + +INLINE constexpr uptr roundDownTo(uptr X, uptr Boundary) { + return X & ~(Boundary - 1); +} + +INLINE constexpr bool isAligned(uptr X, uptr Alignment) { + return (X & (Alignment - 1)) == 0; +} + +template <class T> constexpr T Min(T A, T B) { return A < B ? A : B; } + +template <class T> constexpr T Max(T A, T B) { return A > B ? A : B; } + +template <class T> void Swap(T &A, T &B) { + T Tmp = A; + A = B; + B = Tmp; +} + +INLINE bool isPowerOfTwo(uptr X) { return (X & (X - 1)) == 0; } + +INLINE uptr getMostSignificantSetBitIndex(uptr X) { + DCHECK_NE(X, 0U); + return SCUDO_WORDSIZE - 1U - static_cast<uptr>(__builtin_clzl(X)); +} + +INLINE uptr roundUpToPowerOfTwo(uptr Size) { + DCHECK(Size); + if (isPowerOfTwo(Size)) + return Size; + const uptr Up = getMostSignificantSetBitIndex(Size); + DCHECK_LT(Size, (1UL << (Up + 1))); + DCHECK_GT(Size, (1UL << Up)); + return 1UL << (Up + 1); +} + +INLINE uptr getLeastSignificantSetBitIndex(uptr X) { + DCHECK_NE(X, 0U); + return static_cast<uptr>(__builtin_ctzl(X)); +} + +INLINE uptr getLog2(uptr X) { + DCHECK(isPowerOfTwo(X)); + return getLeastSignificantSetBitIndex(X); +} + +INLINE u32 getRandomU32(u32 *State) { + // ANSI C linear congruential PRNG (16-bit output). + // return (*State = *State * 1103515245 + 12345) >> 16; + // XorShift (32-bit output). + *State ^= *State << 13; + *State ^= *State >> 17; + *State ^= *State << 5; + return *State; +} + +INLINE u32 getRandomModN(u32 *State, u32 N) { + return getRandomU32(State) % N; // [0, N) +} + +template <typename T> INLINE void shuffle(T *A, u32 N, u32 *RandState) { + if (N <= 1) + return; + u32 State = *RandState; + for (u32 I = N - 1; I > 0; I--) + Swap(A[I], A[getRandomModN(&State, I + 1)]); + *RandState = State; +} + +// Hardware specific inlinable functions. + +INLINE void yieldProcessor(u8 Count) { +#if defined(__i386__) || defined(__x86_64__) + __asm__ __volatile__("" ::: "memory"); + for (u8 I = 0; I < Count; I++) + __asm__ __volatile__("pause"); +#elif defined(__aarch64__) || defined(__arm__) + __asm__ __volatile__("" ::: "memory"); + for (u8 I = 0; I < Count; I++) + __asm__ __volatile__("yield"); +#endif + __asm__ __volatile__("" ::: "memory"); +} + +// Platform specific functions. + +extern uptr PageSizeCached; +uptr getPageSizeSlow(); +INLINE uptr getPageSizeCached() { + // Bionic uses a hardcoded value. + if (SCUDO_ANDROID) + return 4096U; + if (LIKELY(PageSizeCached)) + return PageSizeCached; + return getPageSizeSlow(); +} + +u32 getNumberOfCPUs(); + +const char *getEnv(const char *Name); + +u64 getMonotonicTime(); + +// Our randomness gathering function is limited to 256 bytes to ensure we get +// as many bytes as requested, and avoid interruptions (on Linux). +constexpr uptr MaxRandomLength = 256U; +bool getRandom(void *Buffer, uptr Length, bool Blocking = false); + +// Platform memory mapping functions. + +#define MAP_ALLOWNOMEM (1U << 0) +#define MAP_NOACCESS (1U << 1) +#define MAP_RESIZABLE (1U << 2) + +// Our platform memory mapping use is restricted to 3 scenarios: +// - reserve memory at a random address (MAP_NOACCESS); +// - commit memory in a previously reserved space; +// - commit memory at a random address. +// As such, only a subset of parameters combinations is valid, which is checked +// by the function implementation. The Data parameter allows to pass opaque +// platform specific data to the function. +// Returns nullptr on error or dies if MAP_ALLOWNOMEM is not specified. +void *map(void *Addr, uptr Size, const char *Name, uptr Flags = 0, + MapPlatformData *Data = nullptr); + +// Indicates that we are getting rid of the whole mapping, which might have +// further consequences on Data, depending on the platform. +#define UNMAP_ALL (1U << 0) + +void unmap(void *Addr, uptr Size, uptr Flags = 0, + MapPlatformData *Data = nullptr); + +void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, + MapPlatformData *Data = nullptr); + +// Internal map & unmap fatal error. This must not call map(). +void NORETURN dieOnMapUnmapError(bool OutOfMemory = false); + +// Logging related functions. + +void setAbortMessage(const char *Message); + +} // namespace scudo + +#endif // SCUDO_COMMON_H_ diff --git a/lib/scudo/standalone/crc32_hw.cc b/lib/scudo/standalone/crc32_hw.cc new file mode 100644 index 000000000000..f4dae7b5fea8 --- /dev/null +++ b/lib/scudo/standalone/crc32_hw.cc @@ -0,0 +1,19 @@ +//===-- crc32_hw.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "checksum.h" + +namespace scudo { + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +u32 computeHardwareCRC32(u32 Crc, uptr Data) { + return static_cast<u32>(CRC32_INTRINSIC(Crc, Data)); +} +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +} // namespace scudo diff --git a/lib/scudo/standalone/flags.cc b/lib/scudo/standalone/flags.cc new file mode 100644 index 000000000000..21144f211102 --- /dev/null +++ b/lib/scudo/standalone/flags.cc @@ -0,0 +1,57 @@ +//===-- flags.cc ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "flags.h" +#include "common.h" +#include "flags_parser.h" +#include "interface.h" + +namespace scudo { + +Flags *getFlags() { + static Flags F; + return &F; +} + +void Flags::setDefaults() { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "flags.inc" +#undef SCUDO_FLAG +} + +void registerFlags(FlagParser *Parser, Flags *F) { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) \ + Parser->registerFlag(#Name, Description, FlagType::FT_##Type, \ + reinterpret_cast<void *>(&F->Name)); +#include "flags.inc" +#undef SCUDO_FLAG +} + +static const char *getCompileDefinitionScudoDefaultOptions() { +#ifdef SCUDO_DEFAULT_OPTIONS + return STRINGIFY(SCUDO_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +static const char *getScudoDefaultOptions() { + return (&__scudo_default_options) ? __scudo_default_options() : ""; +} + +void initFlags() { + Flags *F = getFlags(); + F->setDefaults(); + FlagParser Parser; + registerFlags(&Parser, F); + Parser.parseString(getCompileDefinitionScudoDefaultOptions()); + Parser.parseString(getScudoDefaultOptions()); + Parser.parseString(getEnv("SCUDO_OPTIONS")); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/flags.h b/lib/scudo/standalone/flags.h new file mode 100644 index 000000000000..edd39a1b8ba9 --- /dev/null +++ b/lib/scudo/standalone/flags.h @@ -0,0 +1,30 @@ +//===-- flags.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAGS_H_ +#define SCUDO_FLAGS_H_ + +#include "internal_defs.h" + +namespace scudo { + +struct Flags { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "flags.inc" +#undef SCUDO_FLAG + void setDefaults(); +}; + +Flags *getFlags(); +void initFlags(); +class FlagParser; +void registerFlags(FlagParser *Parser, Flags *F); + +} // namespace scudo + +#endif // SCUDO_FLAGS_H_ diff --git a/lib/scudo/standalone/flags.inc b/lib/scudo/standalone/flags.inc new file mode 100644 index 000000000000..25b86e14fa94 --- /dev/null +++ b/lib/scudo/standalone/flags.inc @@ -0,0 +1,50 @@ +//===-- flags.inc -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAG +#error "Define SCUDO_FLAG prior to including this file!" +#endif + +SCUDO_FLAG(int, quarantine_size_kb, 0, + "Size (in kilobytes) of quarantine used to delay the actual " + "deallocation of chunks. Lower value may reduce memory usage but " + "decrease the effectiveness of the mitigation.") + +SCUDO_FLAG(int, thread_local_quarantine_size_kb, 0, + "Size (in kilobytes) of per-thread cache used to offload the global " + "quarantine. Lower value may reduce memory usage but might increase " + "the contention on the global quarantine.") + +SCUDO_FLAG(int, quarantine_max_chunk_size, 0, + "Size (in bytes) up to which chunks will be quarantined (if lower " + "than or equal to).") + +SCUDO_FLAG(bool, dealloc_type_mismatch, false, + "Terminate on a type mismatch in allocation-deallocation functions, " + "eg: malloc/delete, new/free, new/delete[], etc.") + +SCUDO_FLAG(bool, delete_size_mismatch, true, + "Terminate on a size mismatch between a sized-delete and the actual " + "size of a chunk (as provided to new/new[]).") + +SCUDO_FLAG(bool, zero_contents, false, "Zero chunk contents on allocation.") + +SCUDO_FLAG(int, rss_limit_mb, -1, + "Enforce an upper limit (in megabytes) to the process RSS. The " + "allocator will terminate or return NULL when allocations are " + "attempted past that limit (depending on may_return_null). Negative " + "values disable the feature.") + +SCUDO_FLAG(bool, may_return_null, true, + "Indicate whether the allocator should terminate instead of " + "returning NULL in otherwise non-fatal error scenarios, eg: OOM, " + "invalid allocation alignments, etc.") + +SCUDO_FLAG(int, release_to_os_interval_ms, 5000, + "Interval (in milliseconds) at which to attempt release of unused " + "memory to the OS. Negative values disable the feature.") diff --git a/lib/scudo/standalone/flags_parser.cc b/lib/scudo/standalone/flags_parser.cc new file mode 100644 index 000000000000..5f1253f58d52 --- /dev/null +++ b/lib/scudo/standalone/flags_parser.cc @@ -0,0 +1,164 @@ +//===-- flags_parser.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "flags_parser.h" +#include "common.h" +#include "report.h" + +#include <stdlib.h> +#include <string.h> + +namespace scudo { + +class UnknownFlagsRegistry { + static const u32 MaxUnknownFlags = 16; + const char *UnknownFlagsNames[MaxUnknownFlags]; + u32 NumberOfUnknownFlags; + +public: + void add(const char *Name) { + CHECK_LT(NumberOfUnknownFlags, MaxUnknownFlags); + UnknownFlagsNames[NumberOfUnknownFlags++] = Name; + } + + void report() { + if (!NumberOfUnknownFlags) + return; + Printf("Scudo WARNING: found %d unrecognized flag(s):\n", + NumberOfUnknownFlags); + for (u32 I = 0; I < NumberOfUnknownFlags; ++I) + Printf(" %s\n", UnknownFlagsNames[I]); + NumberOfUnknownFlags = 0; + } +}; +static UnknownFlagsRegistry UnknownFlags; + +void reportUnrecognizedFlags() { UnknownFlags.report(); } + +void FlagParser::printFlagDescriptions() { + Printf("Available flags for Scudo:\n"); + for (u32 I = 0; I < NumberOfFlags; ++I) + Printf("\t%s\n\t\t- %s\n", Flags[I].Name, Flags[I].Desc); +} + +static bool isSeparator(char C) { + return C == ' ' || C == ',' || C == ':' || C == '\n' || C == '\t' || + C == '\r'; +} + +static bool isSeparatorOrNull(char C) { return !C || isSeparator(C); } + +void FlagParser::skipWhitespace() { + while (isSeparator(Buffer[Pos])) + ++Pos; +} + +void FlagParser::parseFlag() { + const uptr NameStart = Pos; + while (Buffer[Pos] != '=' && !isSeparatorOrNull(Buffer[Pos])) + ++Pos; + if (Buffer[Pos] != '=') + reportError("expected '='"); + const char *Name = Buffer + NameStart; + const uptr ValueStart = ++Pos; + const char *Value; + if (Buffer[Pos] == '\'' || Buffer[Pos] == '"') { + const char Quote = Buffer[Pos++]; + while (Buffer[Pos] != 0 && Buffer[Pos] != Quote) + ++Pos; + if (Buffer[Pos] == 0) + reportError("unterminated string"); + Value = Buffer + ValueStart + 1; + ++Pos; // consume the closing quote + } else { + while (!isSeparatorOrNull(Buffer[Pos])) + ++Pos; + Value = Buffer + ValueStart; + } + if (!runHandler(Name, Value)) + reportError("flag parsing failed."); +} + +void FlagParser::parseFlags() { + while (true) { + skipWhitespace(); + if (Buffer[Pos] == 0) + break; + parseFlag(); + } +} + +void FlagParser::parseString(const char *S) { + if (!S) + return; + // Backup current parser state to allow nested parseString() calls. + const char *OldBuffer = Buffer; + const uptr OldPos = Pos; + Buffer = S; + Pos = 0; + + parseFlags(); + + Buffer = OldBuffer; + Pos = OldPos; +} + +INLINE bool parseBool(const char *Value, bool *b) { + if (strncmp(Value, "0", 1) == 0 || strncmp(Value, "no", 2) == 0 || + strncmp(Value, "false", 5) == 0) { + *b = false; + return true; + } + if (strncmp(Value, "1", 1) == 0 || strncmp(Value, "yes", 3) == 0 || + strncmp(Value, "true", 4) == 0) { + *b = true; + return true; + } + return false; +} + +bool FlagParser::runHandler(const char *Name, const char *Value) { + for (u32 I = 0; I < NumberOfFlags; ++I) { + const uptr Len = strlen(Flags[I].Name); + if (strncmp(Name, Flags[I].Name, Len) != 0 || Name[Len] != '=') + continue; + bool Ok = false; + switch (Flags[I].Type) { + case FlagType::FT_bool: + Ok = parseBool(Value, reinterpret_cast<bool *>(Flags[I].Var)); + if (!Ok) + reportInvalidFlag("bool", Value); + break; + case FlagType::FT_int: + char *ValueEnd; + *reinterpret_cast<int *>(Flags[I].Var) = + static_cast<int>(strtol(Value, &ValueEnd, 10)); + Ok = + *ValueEnd == '"' || *ValueEnd == '\'' || isSeparatorOrNull(*ValueEnd); + if (!Ok) + reportInvalidFlag("int", Value); + break; + } + return Ok; + } + // Unrecognized flag. This is not a fatal error, we may print a warning later. + UnknownFlags.add(Name); + return true; +} + +void FlagParser::registerFlag(const char *Name, const char *Desc, FlagType Type, + void *Var) { + CHECK_LT(NumberOfFlags, MaxFlags); + Flags[NumberOfFlags].Name = Name; + Flags[NumberOfFlags].Desc = Desc; + Flags[NumberOfFlags].Type = Type; + Flags[NumberOfFlags].Var = Var; + ++NumberOfFlags; +} + +} // namespace scudo diff --git a/lib/scudo/standalone/flags_parser.h b/lib/scudo/standalone/flags_parser.h new file mode 100644 index 000000000000..857b50e880ec --- /dev/null +++ b/lib/scudo/standalone/flags_parser.h @@ -0,0 +1,55 @@ +//===-- flags_parser.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAGS_PARSER_H_ +#define SCUDO_FLAGS_PARSER_H_ + +#include "report.h" +#include "string_utils.h" + +#include <stddef.h> + +namespace scudo { + +enum class FlagType : u8 { + FT_bool, + FT_int, +}; + +class FlagParser { +public: + void registerFlag(const char *Name, const char *Desc, FlagType Type, + void *Var); + void parseString(const char *S); + void printFlagDescriptions(); + +private: + static const u32 MaxFlags = 12; + struct Flag { + const char *Name; + const char *Desc; + FlagType Type; + void *Var; + } Flags[MaxFlags]; + + u32 NumberOfFlags = 0; + const char *Buffer = nullptr; + uptr Pos = 0; + + void reportFatalError(const char *Error); + void skipWhitespace(); + void parseFlags(); + void parseFlag(); + bool runHandler(const char *Name, const char *Value); +}; + +void reportUnrecognizedFlags(); + +} // namespace scudo + +#endif // SCUDO_FLAGS_PARSER_H_ diff --git a/lib/scudo/standalone/fuchsia.cc b/lib/scudo/standalone/fuchsia.cc new file mode 100644 index 000000000000..896d346e7e72 --- /dev/null +++ b/lib/scudo/standalone/fuchsia.cc @@ -0,0 +1,189 @@ +//===-- fuchsia.cc ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_FUCHSIA + +#include "common.h" +#include "mutex.h" +#include "string_utils.h" + +#include <lib/sync/mutex.h> // for sync_mutex_t +#include <limits.h> // for PAGE_SIZE +#include <stdlib.h> // for getenv() +#include <zircon/compiler.h> +#include <zircon/sanitizer.h> +#include <zircon/syscalls.h> + +namespace scudo { + +uptr getPageSize() { return PAGE_SIZE; } + +void NORETURN die() { __builtin_trap(); } + +// We zero-initialize the Extra parameter of map(), make sure this is consistent +// with ZX_HANDLE_INVALID. +COMPILER_CHECK(ZX_HANDLE_INVALID == 0); + +static void *allocateVmar(uptr Size, MapPlatformData *Data, bool AllowNoMem) { + // Only scenario so far. + DCHECK(Data); + DCHECK_EQ(Data->Vmar, ZX_HANDLE_INVALID); + + const zx_status_t Status = _zx_vmar_allocate( + _zx_vmar_root_self(), + ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0, + Size, &Data->Vmar, &Data->VmarBase); + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + return reinterpret_cast<void *>(Data->VmarBase); +} + +void *map(void *Addr, uptr Size, const char *Name, uptr Flags, + MapPlatformData *Data) { + DCHECK_EQ(Size % PAGE_SIZE, 0); + const bool AllowNoMem = !!(Flags & MAP_ALLOWNOMEM); + + // For MAP_NOACCESS, just allocate a Vmar and return. + if (Flags & MAP_NOACCESS) + return allocateVmar(Size, Data, AllowNoMem); + + const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self(); + CHECK_NE(Vmar, ZX_HANDLE_INVALID); + + zx_status_t Status; + zx_handle_t Vmo; + uint64_t VmoSize = 0; + if (Data && Data->Vmo != ZX_HANDLE_INVALID) { + // If a Vmo was specified, it's a resize operation. + CHECK(Addr); + DCHECK(Flags & MAP_RESIZABLE); + Vmo = Data->Vmo; + VmoSize = Data->VmoSize; + Status = _zx_vmo_set_size(Vmo, VmoSize + Size); + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + } else { + // Otherwise, create a Vmo and set its name. + Status = _zx_vmo_create(Size, ZX_VMO_RESIZABLE, &Vmo); + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + _zx_object_set_property(Vmo, ZX_PROP_NAME, Name, strlen(Name)); + } + + uintptr_t P; + zx_vm_option_t MapFlags = + ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_ALLOW_FAULTS; + const uint64_t Offset = + Addr ? reinterpret_cast<uintptr_t>(Addr) - Data->VmarBase : 0; + if (Offset) + MapFlags |= ZX_VM_SPECIFIC; + Status = _zx_vmar_map(Vmar, MapFlags, Offset, Vmo, VmoSize, Size, &P); + // No need to track the Vmo if we don't intend on resizing it. Close it. + if (Flags & MAP_RESIZABLE) { + DCHECK(Data); + DCHECK_EQ(Data->Vmo, ZX_HANDLE_INVALID); + Data->Vmo = Vmo; + } else { + CHECK_EQ(_zx_handle_close(Vmo), ZX_OK); + } + if (Status != ZX_OK) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY); + return nullptr; + } + if (Data) + Data->VmoSize += Size; + + return reinterpret_cast<void *>(P); +} + +void unmap(void *Addr, uptr Size, uptr Flags, MapPlatformData *Data) { + if (Flags & UNMAP_ALL) { + DCHECK_NE(Data, nullptr); + const zx_handle_t Vmar = Data->Vmar; + DCHECK_NE(Vmar, _zx_vmar_root_self()); + // Destroying the vmar effectively unmaps the whole mapping. + CHECK_EQ(_zx_vmar_destroy(Vmar), ZX_OK); + CHECK_EQ(_zx_handle_close(Vmar), ZX_OK); + } else { + const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self(); + const zx_status_t Status = + _zx_vmar_unmap(Vmar, reinterpret_cast<uintptr_t>(Addr), Size); + if (Status != ZX_OK) + dieOnMapUnmapError(); + } + if (Data) { + if (Data->Vmo != ZX_HANDLE_INVALID) + CHECK_EQ(_zx_handle_close(Data->Vmo), ZX_OK); + memset(Data, 0, sizeof(*Data)); + } +} + +void releasePagesToOS(UNUSED uptr BaseAddress, uptr Offset, uptr Size, + MapPlatformData *Data) { + DCHECK(Data); + DCHECK_NE(Data->Vmar, ZX_HANDLE_INVALID); + DCHECK_NE(Data->Vmo, ZX_HANDLE_INVALID); + const zx_status_t Status = + _zx_vmo_op_range(Data->Vmo, ZX_VMO_OP_DECOMMIT, Offset, Size, NULL, 0); + CHECK_EQ(Status, ZX_OK); +} + +const char *getEnv(const char *Name) { return getenv(Name); } + +// Note: we need to flag these methods with __TA_NO_THREAD_SAFETY_ANALYSIS +// because the Fuchsia implementation of sync_mutex_t has clang thread safety +// annotations. Were we to apply proper capability annotations to the top level +// HybridMutex class itself, they would not be needed. As it stands, the +// thread analysis thinks that we are locking the mutex and accidentally leaving +// it locked on the way out. +bool HybridMutex::tryLock() __TA_NO_THREAD_SAFETY_ANALYSIS { + // Size and alignment must be compatible between both types. + return sync_mutex_trylock(&M) == ZX_OK; +} + +void HybridMutex::lockSlow() __TA_NO_THREAD_SAFETY_ANALYSIS { + sync_mutex_lock(&M); +} + +void HybridMutex::unlock() __TA_NO_THREAD_SAFETY_ANALYSIS { + sync_mutex_unlock(&M); +} + +u64 getMonotonicTime() { return _zx_clock_get_monotonic(); } + +u32 getNumberOfCPUs() { return _zx_system_get_num_cpus(); } + +bool getRandom(void *Buffer, uptr Length, bool Blocking) { + COMPILER_CHECK(MaxRandomLength <= ZX_CPRNG_DRAW_MAX_LEN); + if (!Buffer || !Length || Length > MaxRandomLength) + return false; + _zx_cprng_draw(Buffer, Length); + return true; +} + +void outputRaw(const char *Buffer) { + __sanitizer_log_write(Buffer, strlen(Buffer)); +} + +void setAbortMessage(const char *Message) {} + +} // namespace scudo + +#endif // SCUDO_FUCHSIA diff --git a/lib/scudo/standalone/fuchsia.h b/lib/scudo/standalone/fuchsia.h new file mode 100644 index 000000000000..d6993f892140 --- /dev/null +++ b/lib/scudo/standalone/fuchsia.h @@ -0,0 +1,31 @@ +//===-- fuchsia.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FUCHSIA_H_ +#define SCUDO_FUCHSIA_H_ + +#include "platform.h" + +#if SCUDO_FUCHSIA + +#include <zircon/process.h> + +namespace scudo { + +struct MapPlatformData { + zx_handle_t Vmar; + zx_handle_t Vmo; + uintptr_t VmarBase; + uint64_t VmoSize; +}; + +} // namespace scudo + +#endif // SCUDO_FUCHSIA + +#endif // SCUDO_FUCHSIA_H_ diff --git a/lib/scudo/standalone/interface.h b/lib/scudo/standalone/interface.h new file mode 100644 index 000000000000..e2639823f426 --- /dev/null +++ b/lib/scudo/standalone/interface.h @@ -0,0 +1,29 @@ +//===-- interface.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_INTERFACE_H_ +#define SCUDO_INTERFACE_H_ + +#include "internal_defs.h" + +extern "C" { + +WEAK INTERFACE const char *__scudo_default_options(); + +// Post-allocation & pre-deallocation hooks. +// They must be thread-safe and not use heap related functions. +WEAK INTERFACE void __scudo_allocate_hook(void *ptr, size_t size); +WEAK INTERFACE void __scudo_deallocate_hook(void *ptr); + +WEAK INTERFACE void __scudo_print_stats(void); + +typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); + +} // extern "C" + +#endif // SCUDO_INTERFACE_H_ diff --git a/lib/scudo/standalone/internal_defs.h b/lib/scudo/standalone/internal_defs.h new file mode 100644 index 000000000000..901eac372b36 --- /dev/null +++ b/lib/scudo/standalone/internal_defs.h @@ -0,0 +1,135 @@ +//===-- internal_defs.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_INTERNAL_DEFS_H_ +#define SCUDO_INTERNAL_DEFS_H_ + +#include "platform.h" + +#include <stdint.h> + +#ifndef SCUDO_DEBUG +#define SCUDO_DEBUG 0 +#endif + +#define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0])) + +// String related macros. + +#define STRINGIFY_(S) #S +#define STRINGIFY(S) STRINGIFY_(S) +#define CONCATENATE_(S, C) S##C +#define CONCATENATE(S, C) CONCATENATE_(S, C) + +// Attributes & builtins related macros. + +#define INTERFACE __attribute__((visibility("default"))) +#define WEAK __attribute__((weak)) +#define INLINE inline +#define ALWAYS_INLINE inline __attribute__((always_inline)) +#define ALIAS(X) __attribute__((alias(X))) +// Please only use the ALIGNED macro before the type. Using ALIGNED after the +// variable declaration is not portable. +#define ALIGNED(X) __attribute__((aligned(X))) +#define FORMAT(F, A) __attribute__((format(printf, F, A))) +#define NOINLINE __attribute__((noinline)) +#define NORETURN __attribute__((noreturn)) +#define THREADLOCAL __thread +#define LIKELY(X) __builtin_expect(!!(X), 1) +#define UNLIKELY(X) __builtin_expect(!!(X), 0) +#if defined(__i386__) || defined(__x86_64__) +// __builtin_prefetch(X) generates prefetchnt0 on x86 +#define PREFETCH(X) __asm__("prefetchnta (%0)" : : "r"(X)) +#else +#define PREFETCH(X) __builtin_prefetch(X) +#endif +#define UNUSED __attribute__((unused)) +#define USED __attribute__((used)) +#define NOEXCEPT noexcept + +namespace scudo { + +typedef unsigned long uptr; +typedef signed long sptr; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned long long u64; +typedef signed char s8; +typedef signed short s16; +typedef signed int s32; +typedef signed long long s64; + +// The following two functions have platform specific implementations. +void outputRaw(const char *Buffer); +void NORETURN die(); + +#define RAW_CHECK_MSG(Expr, Msg) \ + do { \ + if (UNLIKELY(!(Expr))) { \ + outputRaw(Msg); \ + die(); \ + } \ + } while (false) + +#define RAW_CHECK(Expr) RAW_CHECK_MSG(Expr, #Expr) + +void NORETURN reportCheckFailed(const char *File, int Line, + const char *Condition, u64 Value1, u64 Value2); + +#define CHECK_IMPL(C1, Op, C2) \ + do { \ + u64 V1 = (u64)(C1); \ + u64 V2 = (u64)(C2); \ + if (UNLIKELY(!(V1 Op V2))) { \ + reportCheckFailed(__FILE__, __LINE__, "(" #C1 ") " #Op " (" #C2 ")", V1, \ + V2); \ + die(); \ + } \ + } while (false) + +#define CHECK(A) CHECK_IMPL((A), !=, 0) +#define CHECK_EQ(A, B) CHECK_IMPL((A), ==, (B)) +#define CHECK_NE(A, B) CHECK_IMPL((A), !=, (B)) +#define CHECK_LT(A, B) CHECK_IMPL((A), <, (B)) +#define CHECK_LE(A, B) CHECK_IMPL((A), <=, (B)) +#define CHECK_GT(A, B) CHECK_IMPL((A), >, (B)) +#define CHECK_GE(A, B) CHECK_IMPL((A), >=, (B)) + +#if SCUDO_DEBUG +#define DCHECK(A) CHECK(A) +#define DCHECK_EQ(A, B) CHECK_EQ(A, B) +#define DCHECK_NE(A, B) CHECK_NE(A, B) +#define DCHECK_LT(A, B) CHECK_LT(A, B) +#define DCHECK_LE(A, B) CHECK_LE(A, B) +#define DCHECK_GT(A, B) CHECK_GT(A, B) +#define DCHECK_GE(A, B) CHECK_GE(A, B) +#else +#define DCHECK(A) +#define DCHECK_EQ(A, B) +#define DCHECK_NE(A, B) +#define DCHECK_LT(A, B) +#define DCHECK_LE(A, B) +#define DCHECK_GT(A, B) +#define DCHECK_GE(A, B) +#endif + +// The superfluous die() call effectively makes this macro NORETURN. +#define UNREACHABLE(Msg) \ + do { \ + CHECK(0 && Msg); \ + die(); \ + } while (0) + +#define COMPILER_CHECK(Pred) static_assert(Pred, "") + +enum LinkerInitialized { LINKER_INITIALIZED = 0 }; + +} // namespace scudo + +#endif // SCUDO_INTERNAL_DEFS_H_ diff --git a/lib/scudo/standalone/linux.cc b/lib/scudo/standalone/linux.cc new file mode 100644 index 000000000000..049477bba8b0 --- /dev/null +++ b/lib/scudo/standalone/linux.cc @@ -0,0 +1,171 @@ +//===-- linux.cc ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX + +#include "common.h" +#include "linux.h" +#include "mutex.h" +#include "string_utils.h" + +#include <errno.h> +#include <fcntl.h> +#include <linux/futex.h> +#include <sched.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <sys/time.h> +#include <time.h> +#include <unistd.h> + +#if SCUDO_ANDROID +#include <sys/prctl.h> +// Definitions of prctl arguments to set a vma name in Android kernels. +#define ANDROID_PR_SET_VMA 0x53564d41 +#define ANDROID_PR_SET_VMA_ANON_NAME 0 +#endif + +namespace scudo { + +uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); } + +void NORETURN die() { abort(); } + +void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, + UNUSED MapPlatformData *Data) { + int MmapFlags = MAP_PRIVATE | MAP_ANON; + int MmapProt; + if (Flags & MAP_NOACCESS) { + MmapFlags |= MAP_NORESERVE; + MmapProt = PROT_NONE; + } else { + MmapProt = PROT_READ | PROT_WRITE; + } + if (Addr) { + // Currently no scenario for a noaccess mapping with a fixed address. + DCHECK_EQ(Flags & MAP_NOACCESS, 0); + MmapFlags |= MAP_FIXED; + } + void *P = mmap(Addr, Size, MmapProt, MmapFlags, -1, 0); + if (P == MAP_FAILED) { + if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM) + dieOnMapUnmapError(errno == ENOMEM); + return nullptr; + } +#if SCUDO_ANDROID + if (!(Flags & MAP_NOACCESS)) + prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name); +#endif + return P; +} + +void unmap(void *Addr, uptr Size, UNUSED uptr Flags, + UNUSED MapPlatformData *Data) { + if (munmap(Addr, Size) != 0) + dieOnMapUnmapError(); +} + +void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, + UNUSED MapPlatformData *Data) { + void *Addr = reinterpret_cast<void *>(BaseAddress + Offset); + while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) { + } +} + +// Calling getenv should be fine (c)(tm) at any time. +const char *getEnv(const char *Name) { return getenv(Name); } + +namespace { +enum State : u32 { Unlocked = 0, Locked = 1, Sleeping = 2 }; +} + +bool HybridMutex::tryLock() { + return atomic_compare_exchange(&M, Unlocked, Locked) == Unlocked; +} + +// The following is based on https://akkadia.org/drepper/futex.pdf. +void HybridMutex::lockSlow() { + u32 V = atomic_compare_exchange(&M, Unlocked, Locked); + if (V == Unlocked) + return; + if (V != Sleeping) + V = atomic_exchange(&M, Sleeping, memory_order_acquire); + while (V != Unlocked) { + syscall(SYS_futex, reinterpret_cast<uptr>(&M), FUTEX_WAIT_PRIVATE, Sleeping, + nullptr, nullptr, 0); + V = atomic_exchange(&M, Sleeping, memory_order_acquire); + } +} + +void HybridMutex::unlock() { + if (atomic_fetch_sub(&M, 1U, memory_order_release) != Locked) { + atomic_store(&M, Unlocked, memory_order_release); + syscall(SYS_futex, reinterpret_cast<uptr>(&M), FUTEX_WAKE_PRIVATE, 1, + nullptr, nullptr, 0); + } +} + +u64 getMonotonicTime() { + timespec TS; + clock_gettime(CLOCK_MONOTONIC, &TS); + return static_cast<u64>(TS.tv_sec) * (1000ULL * 1000 * 1000) + + static_cast<u64>(TS.tv_nsec); +} + +u32 getNumberOfCPUs() { + cpu_set_t CPUs; + CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); + return static_cast<u32>(CPU_COUNT(&CPUs)); +} + +// Blocking is possibly unused if the getrandom block is not compiled in. +bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) { + if (!Buffer || !Length || Length > MaxRandomLength) + return false; + ssize_t ReadBytes; +#if defined(SYS_getrandom) +#if !defined(GRND_NONBLOCK) +#define GRND_NONBLOCK 1 +#endif + // Up to 256 bytes, getrandom will not be interrupted. + ReadBytes = + syscall(SYS_getrandom, Buffer, Length, Blocking ? 0 : GRND_NONBLOCK); + if (ReadBytes == static_cast<ssize_t>(Length)) + return true; +#endif // defined(SYS_getrandom) + // Up to 256 bytes, a read off /dev/urandom will not be interrupted. + // Blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom. + const int FileDesc = open("/dev/urandom", O_RDONLY); + if (FileDesc == -1) + return false; + ReadBytes = read(FileDesc, Buffer, Length); + close(FileDesc); + return (ReadBytes == static_cast<ssize_t>(Length)); +} + +void outputRaw(const char *Buffer) { + static HybridMutex Mutex; + ScopedLock L(Mutex); + write(2, Buffer, strlen(Buffer)); +} + +extern "C" WEAK void android_set_abort_message(const char *); + +void setAbortMessage(const char *Message) { + if (&android_set_abort_message) + android_set_abort_message(Message); +} + +} // namespace scudo + +#endif // SCUDO_LINUX diff --git a/lib/scudo/standalone/linux.h b/lib/scudo/standalone/linux.h new file mode 100644 index 000000000000..92c9eb5e97ee --- /dev/null +++ b/lib/scudo/standalone/linux.h @@ -0,0 +1,70 @@ +//===-- linux.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_LINUX_H_ +#define SCUDO_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX + +namespace scudo { + +// MapPlatformData is unused on Linux, define it as a minimally sized structure. +struct MapPlatformData {}; + +#if SCUDO_ANDROID + +#if defined(__aarch64__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("mrs %0, tpidr_el0" : "=r"(__v)); \ + __v; \ + }) +#elif defined(__arm__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); \ + __v; \ + }) +#elif defined(__i386__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("movl %%gs:0, %0" : "=r"(__v)); \ + __v; \ + }) +#elif defined(__x86_64__) +#define __get_tls() \ + ({ \ + void **__v; \ + __asm__("mov %%fs:0, %0" : "=r"(__v)); \ + __v; \ + }) +#else +#error "Unsupported architecture." +#endif + +// The Android Bionic team has allocated a TLS slot for sanitizers starting +// with Q, given that Android currently doesn't support ELF TLS. It is used to +// store sanitizer thread specific data. +static const int TLS_SLOT_SANITIZER = 8; // TODO(kostyak): 6 for Q!! + +ALWAYS_INLINE uptr *getAndroidTlsPtr() { + return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]); +} + +#endif // SCUDO_ANDROID + +} // namespace scudo + +#endif // SCUDO_LINUX + +#endif // SCUDO_LINUX_H_ diff --git a/lib/scudo/standalone/list.h b/lib/scudo/standalone/list.h new file mode 100644 index 000000000000..139e73eff5ad --- /dev/null +++ b/lib/scudo/standalone/list.h @@ -0,0 +1,156 @@ +//===-- list.h --------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_LIST_H_ +#define SCUDO_LIST_H_ + +#include "internal_defs.h" + +namespace scudo { + +// Intrusive POD singly-linked list. +// An object with all zero fields should represent a valid empty list. clear() +// should be called on all non-zero-initialized objects before using. +template <class Item> struct IntrusiveList { + friend class Iterator; + + void clear() { + First = Last = nullptr; + Size = 0; + } + + bool empty() const { return Size == 0; } + uptr size() const { return Size; } + + void push_back(Item *X) { + if (empty()) { + X->Next = nullptr; + First = Last = X; + Size = 1; + } else { + X->Next = nullptr; + Last->Next = X; + Last = X; + Size++; + } + } + + void push_front(Item *X) { + if (empty()) { + X->Next = nullptr; + First = Last = X; + Size = 1; + } else { + X->Next = First; + First = X; + Size++; + } + } + + void pop_front() { + DCHECK(!empty()); + First = First->Next; + if (!First) + Last = nullptr; + Size--; + } + + void extract(Item *Prev, Item *X) { + DCHECK(!empty()); + DCHECK_NE(Prev, nullptr); + DCHECK_NE(X, nullptr); + DCHECK_EQ(Prev->Next, X); + Prev->Next = X->Next; + if (Last == X) + Last = Prev; + Size--; + } + + Item *front() { return First; } + const Item *front() const { return First; } + Item *back() { return Last; } + const Item *back() const { return Last; } + + void append_front(IntrusiveList<Item> *L) { + DCHECK_NE(this, L); + if (L->empty()) + return; + if (empty()) { + *this = *L; + } else if (!L->empty()) { + L->Last->Next = First; + First = L->First; + Size += L->size(); + } + L->clear(); + } + + void append_back(IntrusiveList<Item> *L) { + DCHECK_NE(this, L); + if (L->empty()) + return; + if (empty()) { + *this = *L; + } else { + Last->Next = L->First; + Last = L->Last; + Size += L->size(); + } + L->clear(); + } + + void checkConsistency() { + if (Size == 0) { + CHECK_EQ(First, 0); + CHECK_EQ(Last, 0); + } else { + uptr count = 0; + for (Item *I = First;; I = I->Next) { + count++; + if (I == Last) + break; + } + CHECK_EQ(size(), count); + CHECK_EQ(Last->Next, 0); + } + } + + template <class ItemT> class IteratorBase { + public: + explicit IteratorBase(ItemT *CurrentItem) : Current(CurrentItem) {} + IteratorBase &operator++() { + Current = Current->Next; + return *this; + } + bool operator!=(IteratorBase Other) const { + return Current != Other.Current; + } + ItemT &operator*() { return *Current; } + + private: + ItemT *Current; + }; + + typedef IteratorBase<Item> Iterator; + typedef IteratorBase<const Item> ConstIterator; + + Iterator begin() { return Iterator(First); } + Iterator end() { return Iterator(nullptr); } + + ConstIterator begin() const { return ConstIterator(First); } + ConstIterator end() const { return ConstIterator(nullptr); } + +private: + uptr Size; + Item *First; + Item *Last; +}; + +} // namespace scudo + +#endif // SCUDO_LIST_H_ diff --git a/lib/scudo/standalone/local_cache.h b/lib/scudo/standalone/local_cache.h new file mode 100644 index 000000000000..2acc28874015 --- /dev/null +++ b/lib/scudo/standalone/local_cache.h @@ -0,0 +1,181 @@ +//===-- local_cache.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_LOCAL_CACHE_H_ +#define SCUDO_LOCAL_CACHE_H_ + +#include "internal_defs.h" +#include "report.h" +#include "stats.h" + +namespace scudo { + +template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache { + typedef typename SizeClassAllocator::SizeClassMap SizeClassMap; + + struct TransferBatch { + static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint; + void setFromArray(void **Array, u32 N) { + DCHECK_LE(N, MaxNumCached); + for (u32 I = 0; I < N; I++) + Batch[I] = Array[I]; + Count = N; + } + void clear() { Count = 0; } + void add(void *P) { + DCHECK_LT(Count, MaxNumCached); + Batch[Count++] = P; + } + void copyToArray(void **Array) const { + for (u32 I = 0; I < Count; I++) + Array[I] = Batch[I]; + } + u32 getCount() const { return Count; } + void *get(u32 I) const { + DCHECK_LE(I, Count); + return Batch[I]; + } + static u32 getMaxCached(uptr Size) { + return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size)); + } + TransferBatch *Next; + + private: + u32 Count; + void *Batch[MaxNumCached]; + }; + + void initLinkerInitialized(GlobalStats *S, SizeClassAllocator *A) { + Stats.initLinkerInitialized(); + if (S) + S->link(&Stats); + Allocator = A; + } + + void init(GlobalStats *S, SizeClassAllocator *A) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(S, A); + } + + void destroy(GlobalStats *S) { + drain(); + if (S) + S->unlink(&Stats); + } + + void *allocate(uptr ClassId) { + CHECK_LT(ClassId, NumClasses); + PerClass *C = &PerClassArray[ClassId]; + if (C->Count == 0) { + if (UNLIKELY(!refill(C, ClassId))) + return nullptr; + DCHECK_GT(C->Count, 0); + } + // We read ClassSize first before accessing Chunks because it's adjacent to + // Count, while Chunks might be further off (depending on Count). That keeps + // the memory accesses in close quarters. + const uptr ClassSize = C->ClassSize; + void *P = C->Chunks[--C->Count]; + // The jury is still out as to whether any kind of PREFETCH here increases + // performance. It definitely decreases performance on Android though. + // if (!SCUDO_ANDROID) PREFETCH(P); + Stats.add(StatAllocated, ClassSize); + return P; + } + + void deallocate(uptr ClassId, void *P) { + CHECK_LT(ClassId, NumClasses); + PerClass *C = &PerClassArray[ClassId]; + // We still have to initialize the cache in the event that the first heap + // operation in a thread is a deallocation. + initCacheMaybe(C); + if (C->Count == C->MaxCount) + drain(C, ClassId); + // See comment in allocate() about memory accesses. + const uptr ClassSize = C->ClassSize; + C->Chunks[C->Count++] = P; + Stats.sub(StatAllocated, ClassSize); + } + + void drain() { + for (uptr I = 0; I < NumClasses; I++) { + PerClass *C = &PerClassArray[I]; + while (C->Count > 0) + drain(C, I); + } + } + + TransferBatch *createBatch(uptr ClassId, void *B) { + if (ClassId != SizeClassMap::BatchClassId) + B = allocate(SizeClassMap::BatchClassId); + return reinterpret_cast<TransferBatch *>(B); + } + + LocalStats &getStats() { return Stats; } + +private: + static const uptr NumClasses = SizeClassMap::NumClasses; + struct PerClass { + u32 Count; + u32 MaxCount; + uptr ClassSize; + void *Chunks[2 * TransferBatch::MaxNumCached]; + }; + PerClass PerClassArray[NumClasses]; + LocalStats Stats; + SizeClassAllocator *Allocator; + + ALWAYS_INLINE void initCacheMaybe(PerClass *C) { + if (LIKELY(C->MaxCount)) + return; + initCache(); + DCHECK_NE(C->MaxCount, 0U); + } + + NOINLINE void initCache() { + for (uptr I = 0; I < NumClasses; I++) { + PerClass *P = &PerClassArray[I]; + const uptr Size = SizeClassAllocator::getSizeByClassId(I); + P->MaxCount = 2 * TransferBatch::getMaxCached(Size); + P->ClassSize = Size; + } + } + + void destroyBatch(uptr ClassId, void *B) { + if (ClassId != SizeClassMap::BatchClassId) + deallocate(SizeClassMap::BatchClassId, B); + } + + NOINLINE bool refill(PerClass *C, uptr ClassId) { + initCacheMaybe(C); + TransferBatch *B = Allocator->popBatch(this, ClassId); + if (UNLIKELY(!B)) + return false; + DCHECK_GT(B->getCount(), 0); + B->copyToArray(C->Chunks); + C->Count = B->getCount(); + destroyBatch(ClassId, B); + return true; + } + + NOINLINE void drain(PerClass *C, uptr ClassId) { + const u32 Count = Min(C->MaxCount / 2, C->Count); + const uptr FirstIndexToDrain = C->Count - Count; + TransferBatch *B = createBatch(ClassId, C->Chunks[FirstIndexToDrain]); + if (UNLIKELY(!B)) + reportOutOfMemory( + SizeClassAllocator::getSizeByClassId(SizeClassMap::BatchClassId)); + B->setFromArray(&C->Chunks[FirstIndexToDrain], Count); + C->Count -= Count; + Allocator->pushBatch(ClassId, B); + } +}; + +} // namespace scudo + +#endif // SCUDO_LOCAL_CACHE_H_ diff --git a/lib/scudo/standalone/mutex.h b/lib/scudo/standalone/mutex.h new file mode 100644 index 000000000000..b6dc9188d347 --- /dev/null +++ b/lib/scudo/standalone/mutex.h @@ -0,0 +1,73 @@ +//===-- mutex.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_MUTEX_H_ +#define SCUDO_MUTEX_H_ + +#include "atomic_helpers.h" +#include "common.h" + +#include <string.h> + +#if SCUDO_FUCHSIA +#include <lib/sync/mutex.h> // for sync_mutex_t +#endif + +namespace scudo { + +class HybridMutex { +public: + void init() { memset(this, 0, sizeof(*this)); } + bool tryLock(); + NOINLINE void lock() { + if (tryLock()) + return; + // The compiler may try to fully unroll the loop, ending up in a + // NumberOfTries*NumberOfYields block of pauses mixed with tryLocks. This + // is large, ugly and unneeded, a compact loop is better for our purpose + // here. Use a pragma to tell the compiler not to unroll the loop. +#ifdef __clang__ +#pragma nounroll +#endif + for (u8 I = 0U; I < NumberOfTries; I++) { + yieldProcessor(NumberOfYields); + if (tryLock()) + return; + } + lockSlow(); + } + void unlock(); + +private: + static constexpr u8 NumberOfTries = 10U; + static constexpr u8 NumberOfYields = 10U; + +#if SCUDO_LINUX + atomic_u32 M; +#elif SCUDO_FUCHSIA + sync_mutex_t M; +#endif + + void lockSlow(); +}; + +class ScopedLock { +public: + explicit ScopedLock(HybridMutex &M) : Mutex(M) { Mutex.lock(); } + ~ScopedLock() { Mutex.unlock(); } + +private: + HybridMutex &Mutex; + + ScopedLock(const ScopedLock &) = delete; + void operator=(const ScopedLock &) = delete; +}; + +} // namespace scudo + +#endif // SCUDO_MUTEX_H_ diff --git a/lib/scudo/standalone/platform.h b/lib/scudo/standalone/platform.h new file mode 100644 index 000000000000..a897a566f9bf --- /dev/null +++ b/lib/scudo/standalone/platform.h @@ -0,0 +1,70 @@ +//===-- platform.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PLATFORM_H_ +#define SCUDO_PLATFORM_H_ + +#if defined(__linux__) +#define SCUDO_LINUX 1 +#else +#define SCUDO_LINUX 0 +#endif + +#if defined(__ANDROID__) +#define SCUDO_ANDROID 1 +#else +#define SCUDO_ANDROID 0 +#endif + +#if defined(__Fuchsia__) +#define SCUDO_FUCHSIA 1 +#else +#define SCUDO_FUCHSIA 0 +#endif + +#if __LP64__ +#define SCUDO_WORDSIZE 64U +#else +#define SCUDO_WORDSIZE 32U +#endif + +#if SCUDO_WORDSIZE == 64U +#define FIRST_32_SECOND_64(a, b) (b) +#else +#define FIRST_32_SECOND_64(a, b) (a) +#endif + +#ifndef SCUDO_CAN_USE_PRIMARY64 +#define SCUDO_CAN_USE_PRIMARY64 (SCUDO_WORDSIZE == 64U) +#endif + +#ifndef SCUDO_MIN_ALIGNMENT_LOG +// We force malloc-type functions to be aligned to std::max_align_t, but there +// is no reason why the minimum alignment for all other functions can't be 8 +// bytes. Except obviously for applications making incorrect assumptions. +// TODO(kostyak): define SCUDO_MIN_ALIGNMENT_LOG 3 +#define SCUDO_MIN_ALIGNMENT_LOG FIRST_32_SECOND_64(3, 4) +#endif + +#if defined(__aarch64__) +#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) +#else +#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) +#endif + +// Older gcc have issues aligning to a constexpr, and require an integer. +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others. +#if defined(__powerpc__) || defined(__powerpc64__) +#define SCUDO_CACHE_LINE_SIZE 128 +#else +#define SCUDO_CACHE_LINE_SIZE 64 +#endif + +#define SCUDO_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) + +#endif // SCUDO_PLATFORM_H_ diff --git a/lib/scudo/standalone/primary32.h b/lib/scudo/standalone/primary32.h new file mode 100644 index 000000000000..2b2fa8b3d793 --- /dev/null +++ b/lib/scudo/standalone/primary32.h @@ -0,0 +1,401 @@ +//===-- primary32.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PRIMARY32_H_ +#define SCUDO_PRIMARY32_H_ + +#include "bytemap.h" +#include "common.h" +#include "list.h" +#include "local_cache.h" +#include "release.h" +#include "report.h" +#include "stats.h" +#include "string_utils.h" + +namespace scudo { + +// SizeClassAllocator32 is an allocator for 32 or 64-bit address space. +// +// It maps Regions of 2^RegionSizeLog bytes aligned on a 2^RegionSizeLog bytes +// boundary, and keeps a bytemap of the mappable address space to track the size +// class they are associated with. +// +// Mapped regions are split into equally sized Blocks according to the size +// class they belong to, and the associated pointers are shuffled to prevent any +// predictable address pattern (the predictability increases with the block +// size). +// +// Regions for size class 0 are special and used to hold TransferBatches, which +// allow to transfer arrays of pointers from the global size class freelist to +// the thread specific freelist for said class, and back. +// +// Memory used by this allocator is never unmapped but can be partially +// reclaimed if the platform allows for it. + +template <class SizeClassMapT, uptr RegionSizeLog> class SizeClassAllocator32 { +public: + typedef SizeClassMapT SizeClassMap; + // Regions should be large enough to hold the largest Block. + COMPILER_CHECK((1UL << RegionSizeLog) >= SizeClassMap::MaxSize); + typedef SizeClassAllocator32<SizeClassMapT, RegionSizeLog> ThisT; + typedef SizeClassAllocatorLocalCache<ThisT> CacheT; + typedef typename CacheT::TransferBatch TransferBatch; + + static uptr getSizeByClassId(uptr ClassId) { + return (ClassId == SizeClassMap::BatchClassId) + ? sizeof(TransferBatch) + : SizeClassMap::getSizeByClassId(ClassId); + } + + static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } + + void initLinkerInitialized(s32 ReleaseToOsInterval) { + if (SCUDO_FUCHSIA) + reportError("SizeClassAllocator32 is not supported on Fuchsia"); + + PossibleRegions.initLinkerInitialized(); + MinRegionIndex = NumRegions; // MaxRegionIndex is already initialized to 0. + + u32 Seed; + if (UNLIKELY(!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed)))) + Seed = + static_cast<u32>(getMonotonicTime() ^ + (reinterpret_cast<uptr>(SizeClassInfoArray) >> 6)); + const uptr PageSize = getPageSizeCached(); + for (uptr I = 0; I < NumClasses; I++) { + SizeClassInfo *Sci = getSizeClassInfo(I); + Sci->RandState = getRandomU32(&Seed); + // See comment in the 64-bit primary about releasing smaller size classes. + Sci->CanRelease = (ReleaseToOsInterval > 0) && + (I != SizeClassMap::BatchClassId) && + (getSizeByClassId(I) >= (PageSize / 32)); + } + ReleaseToOsIntervalMs = ReleaseToOsInterval; + } + void init(s32 ReleaseToOsInterval) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(ReleaseToOsInterval); + } + + void unmapTestOnly() { + while (NumberOfStashedRegions > 0) + unmap(reinterpret_cast<void *>(RegionsStash[--NumberOfStashedRegions]), + RegionSize); + // TODO(kostyak): unmap the TransferBatch regions as well. + for (uptr I = 0; I < NumRegions; I++) + if (PossibleRegions[I]) + unmap(reinterpret_cast<void *>(I * RegionSize), RegionSize); + PossibleRegions.unmapTestOnly(); + } + + TransferBatch *popBatch(CacheT *C, uptr ClassId) { + DCHECK_LT(ClassId, NumClasses); + SizeClassInfo *Sci = getSizeClassInfo(ClassId); + ScopedLock L(Sci->Mutex); + TransferBatch *B = Sci->FreeList.front(); + if (B) + Sci->FreeList.pop_front(); + else { + B = populateFreeList(C, ClassId, Sci); + if (UNLIKELY(!B)) + return nullptr; + } + DCHECK_GT(B->getCount(), 0); + Sci->Stats.PoppedBlocks += B->getCount(); + return B; + } + + void pushBatch(uptr ClassId, TransferBatch *B) { + DCHECK_LT(ClassId, NumClasses); + DCHECK_GT(B->getCount(), 0); + SizeClassInfo *Sci = getSizeClassInfo(ClassId); + ScopedLock L(Sci->Mutex); + Sci->FreeList.push_front(B); + Sci->Stats.PushedBlocks += B->getCount(); + if (Sci->CanRelease) + releaseToOSMaybe(Sci, ClassId); + } + + void disable() { + for (uptr I = 0; I < NumClasses; I++) + getSizeClassInfo(I)->Mutex.lock(); + } + + void enable() { + for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) + getSizeClassInfo(I)->Mutex.unlock(); + } + + template <typename F> void iterateOverBlocks(F Callback) { + for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++) + if (PossibleRegions[I]) { + const uptr BlockSize = getSizeByClassId(PossibleRegions[I]); + const uptr From = I * RegionSize; + const uptr To = From + (RegionSize / BlockSize) * BlockSize; + for (uptr Block = From; Block < To; Block += BlockSize) + Callback(Block); + } + } + + void printStats() { + // TODO(kostyak): get the RSS per region. + uptr TotalMapped = 0; + uptr PoppedBlocks = 0; + uptr PushedBlocks = 0; + for (uptr I = 0; I < NumClasses; I++) { + SizeClassInfo *Sci = getSizeClassInfo(I); + TotalMapped += Sci->AllocatedUser; + PoppedBlocks += Sci->Stats.PoppedBlocks; + PushedBlocks += Sci->Stats.PushedBlocks; + } + Printf("Stats: SizeClassAllocator32: %zuM mapped in %zu allocations; " + "remains %zu\n", + TotalMapped >> 20, PoppedBlocks, PoppedBlocks - PushedBlocks); + for (uptr I = 0; I < NumClasses; I++) + printStats(I, 0); + } + + void releaseToOS() { + for (uptr I = 0; I < NumClasses; I++) { + if (I == SizeClassMap::BatchClassId) + continue; + SizeClassInfo *Sci = getSizeClassInfo(I); + ScopedLock L(Sci->Mutex); + releaseToOSMaybe(Sci, I, /*Force=*/true); + } + } + +private: + static const uptr NumClasses = SizeClassMap::NumClasses; + static const uptr RegionSize = 1UL << RegionSizeLog; + static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> RegionSizeLog; +#if SCUDO_WORDSIZE == 32U + typedef FlatByteMap<NumRegions> ByteMap; +#else + typedef TwoLevelByteMap<(NumRegions >> 12), 1UL << 12> ByteMap; +#endif + + struct SizeClassStats { + uptr PoppedBlocks; + uptr PushedBlocks; + }; + + struct ReleaseToOsInfo { + uptr PushedBlocksAtLastRelease; + uptr RangesReleased; + uptr LastReleasedBytes; + u64 LastReleaseAtNs; + }; + + struct ALIGNED(SCUDO_CACHE_LINE_SIZE) SizeClassInfo { + HybridMutex Mutex; + IntrusiveList<TransferBatch> FreeList; + SizeClassStats Stats; + bool CanRelease; + u32 RandState; + uptr AllocatedUser; + ReleaseToOsInfo ReleaseInfo; + }; + COMPILER_CHECK(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0); + + uptr computeRegionId(uptr Mem) { + const uptr Id = Mem >> RegionSizeLog; + CHECK_LT(Id, NumRegions); + return Id; + } + + uptr allocateRegionSlow() { + uptr MapSize = 2 * RegionSize; + const uptr MapBase = reinterpret_cast<uptr>( + map(nullptr, MapSize, "scudo:primary", MAP_ALLOWNOMEM)); + if (UNLIKELY(!MapBase)) + return 0; + const uptr MapEnd = MapBase + MapSize; + uptr Region = MapBase; + if (isAligned(Region, RegionSize)) { + ScopedLock L(RegionsStashMutex); + if (NumberOfStashedRegions < MaxStashedRegions) + RegionsStash[NumberOfStashedRegions++] = MapBase + RegionSize; + else + MapSize = RegionSize; + } else { + Region = roundUpTo(MapBase, RegionSize); + unmap(reinterpret_cast<void *>(MapBase), Region - MapBase); + MapSize = RegionSize; + } + const uptr End = Region + MapSize; + if (End != MapEnd) + unmap(reinterpret_cast<void *>(End), MapEnd - End); + return Region; + } + + uptr allocateRegion(uptr ClassId) { + DCHECK_LT(ClassId, NumClasses); + uptr Region = 0; + { + ScopedLock L(RegionsStashMutex); + if (NumberOfStashedRegions > 0) + Region = RegionsStash[--NumberOfStashedRegions]; + } + if (!Region) + Region = allocateRegionSlow(); + if (LIKELY(Region)) { + if (ClassId) { + const uptr RegionIndex = computeRegionId(Region); + if (RegionIndex < MinRegionIndex) + MinRegionIndex = RegionIndex; + if (RegionIndex > MaxRegionIndex) + MaxRegionIndex = RegionIndex; + PossibleRegions.set(RegionIndex, static_cast<u8>(ClassId)); + } + } + return Region; + } + + SizeClassInfo *getSizeClassInfo(uptr ClassId) { + DCHECK_LT(ClassId, NumClasses); + return &SizeClassInfoArray[ClassId]; + } + + bool populateBatches(CacheT *C, SizeClassInfo *Sci, uptr ClassId, + TransferBatch **CurrentBatch, u32 MaxCount, + void **PointersArray, u32 Count) { + if (ClassId != SizeClassMap::BatchClassId) + shuffle(PointersArray, Count, &Sci->RandState); + TransferBatch *B = *CurrentBatch; + for (uptr I = 0; I < Count; I++) { + if (B && B->getCount() == MaxCount) { + Sci->FreeList.push_back(B); + B = nullptr; + } + if (!B) { + B = C->createBatch(ClassId, PointersArray[I]); + if (UNLIKELY(!B)) + return false; + B->clear(); + } + B->add(PointersArray[I]); + } + *CurrentBatch = B; + return true; + } + + NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, + SizeClassInfo *Sci) { + const uptr Region = allocateRegion(ClassId); + if (UNLIKELY(!Region)) + return nullptr; + C->getStats().add(StatMapped, RegionSize); + const uptr Size = getSizeByClassId(ClassId); + const u32 MaxCount = TransferBatch::getMaxCached(Size); + DCHECK_GT(MaxCount, 0); + const uptr NumberOfBlocks = RegionSize / Size; + DCHECK_GT(NumberOfBlocks, 0); + TransferBatch *B = nullptr; + constexpr uptr ShuffleArraySize = 48; + void *ShuffleArray[ShuffleArraySize]; + u32 Count = 0; + const uptr AllocatedUser = NumberOfBlocks * Size; + for (uptr I = Region; I < Region + AllocatedUser; I += Size) { + ShuffleArray[Count++] = reinterpret_cast<void *>(I); + if (Count == ShuffleArraySize) { + if (UNLIKELY(!populateBatches(C, Sci, ClassId, &B, MaxCount, + ShuffleArray, Count))) + return nullptr; + Count = 0; + } + } + if (Count) { + if (UNLIKELY(!populateBatches(C, Sci, ClassId, &B, MaxCount, ShuffleArray, + Count))) + return nullptr; + } + DCHECK(B); + DCHECK_GT(B->getCount(), 0); + Sci->AllocatedUser += AllocatedUser; + if (Sci->CanRelease) + Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTime(); + return B; + } + + void printStats(uptr ClassId, uptr Rss) { + SizeClassInfo *Sci = getSizeClassInfo(ClassId); + if (Sci->AllocatedUser == 0) + return; + const uptr InUse = Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks; + const uptr AvailableChunks = Sci->AllocatedUser / getSizeByClassId(ClassId); + Printf(" %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu inuse: %6zu" + " avail: %6zu rss: %6zuK\n", + ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10, + Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks, InUse, + AvailableChunks, Rss >> 10); + } + + NOINLINE void releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId, + bool Force = false) { + const uptr BlockSize = getSizeByClassId(ClassId); + const uptr PageSize = getPageSizeCached(); + + CHECK_GE(Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks); + const uptr N = Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks; + if (N * BlockSize < PageSize) + return; // No chance to release anything. + if ((Sci->Stats.PushedBlocks - Sci->ReleaseInfo.PushedBlocksAtLastRelease) * + BlockSize < + PageSize) { + return; // Nothing new to release. + } + + if (!Force) { + const s32 IntervalMs = ReleaseToOsIntervalMs; + if (IntervalMs < 0) + return; + if (Sci->ReleaseInfo.LastReleaseAtNs + IntervalMs * 1000000ULL > + getMonotonicTime()) { + return; // Memory was returned recently. + } + } + + // TODO(kostyak): currently not ideal as we loop over all regions and + // iterate multiple times over the same freelist if a ClassId spans multiple + // regions. But it will have to do for now. + for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++) { + if (PossibleRegions[I] == ClassId) { + ReleaseRecorder Recorder(I * RegionSize); + releaseFreeMemoryToOS(&Sci->FreeList, I * RegionSize, + RegionSize / PageSize, BlockSize, &Recorder); + if (Recorder.getReleasedRangesCount() > 0) { + Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks; + Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); + Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); + } + } + } + Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTime(); + } + + SizeClassInfo SizeClassInfoArray[NumClasses]; + + ByteMap PossibleRegions; + // Keep track of the lowest & highest regions allocated to avoid looping + // through the whole NumRegions. + uptr MinRegionIndex; + uptr MaxRegionIndex; + s32 ReleaseToOsIntervalMs; + // Unless several threads request regions simultaneously from different size + // classes, the stash rarely contains more than 1 entry. + static constexpr uptr MaxStashedRegions = 4; + HybridMutex RegionsStashMutex; + uptr NumberOfStashedRegions; + uptr RegionsStash[MaxStashedRegions]; +}; + +} // namespace scudo + +#endif // SCUDO_PRIMARY32_H_ diff --git a/lib/scudo/standalone/primary64.h b/lib/scudo/standalone/primary64.h new file mode 100644 index 000000000000..035182b33ef4 --- /dev/null +++ b/lib/scudo/standalone/primary64.h @@ -0,0 +1,381 @@ +//===-- primary64.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PRIMARY64_H_ +#define SCUDO_PRIMARY64_H_ + +#include "bytemap.h" +#include "common.h" +#include "list.h" +#include "local_cache.h" +#include "release.h" +#include "stats.h" +#include "string_utils.h" + +namespace scudo { + +// SizeClassAllocator64 is an allocator tuned for 64-bit address space. +// +// It starts by reserving NumClasses * 2^RegionSizeLog bytes, equally divided in +// Regions, specific to each size class. Note that the base of that mapping is +// random (based to the platform specific map() capabilities), and that each +// Region actually starts at a random offset from its base. +// +// Regions are mapped incrementally on demand to fulfill allocation requests, +// those mappings being split into equally sized Blocks based on the size class +// they belong to. The Blocks created are shuffled to prevent predictable +// address patterns (the predictability increases with the size of the Blocks). +// +// The 1st Region (for size class 0) holds the TransferBatches. This is a +// structure used to transfer arrays of available pointers from the class size +// freelist to the thread specific freelist, and back. +// +// The memory used by this allocator is never unmapped, but can be partially +// released it the platform allows for it. + +template <class SizeClassMapT, uptr RegionSizeLog> class SizeClassAllocator64 { +public: + typedef SizeClassMapT SizeClassMap; + typedef SizeClassAllocator64<SizeClassMap, RegionSizeLog> ThisT; + typedef SizeClassAllocatorLocalCache<ThisT> CacheT; + typedef typename CacheT::TransferBatch TransferBatch; + + static uptr getSizeByClassId(uptr ClassId) { + return (ClassId == SizeClassMap::BatchClassId) + ? sizeof(TransferBatch) + : SizeClassMap::getSizeByClassId(ClassId); + } + + static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } + + void initLinkerInitialized(s32 ReleaseToOsInterval) { + // Reserve the space required for the Primary. + PrimaryBase = reinterpret_cast<uptr>( + map(nullptr, PrimarySize, "scudo:primary", MAP_NOACCESS, &Data)); + + RegionInfoArray = reinterpret_cast<RegionInfo *>( + map(nullptr, sizeof(RegionInfo) * NumClasses, "scudo:regioninfo")); + DCHECK_EQ(reinterpret_cast<uptr>(RegionInfoArray) % SCUDO_CACHE_LINE_SIZE, + 0); + + u32 Seed; + if (UNLIKELY(!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed)))) + Seed = static_cast<u32>(getMonotonicTime() ^ (PrimaryBase >> 12)); + const uptr PageSize = getPageSizeCached(); + for (uptr I = 0; I < NumClasses; I++) { + RegionInfo *Region = getRegionInfo(I); + // The actual start of a region is offseted by a random number of pages. + Region->RegionBeg = + getRegionBaseByClassId(I) + (getRandomModN(&Seed, 16) + 1) * PageSize; + // Releasing smaller size classes doesn't necessarily yield to a + // meaningful RSS impact: there are more blocks per page, they are + // randomized around, and thus pages are less likely to be entirely empty. + // On top of this, attempting to release those require more iterations and + // memory accesses which ends up being fairly costly. The current lower + // limit is mostly arbitrary and based on empirical observations. + // TODO(kostyak): make the lower limit a runtime option + Region->CanRelease = (ReleaseToOsInterval > 0) && + (I != SizeClassMap::BatchClassId) && + (getSizeByClassId(I) >= (PageSize / 32)); + Region->RandState = getRandomU32(&Seed); + } + ReleaseToOsIntervalMs = ReleaseToOsInterval; + } + void init(s32 ReleaseToOsInterval) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(ReleaseToOsInterval); + } + + void unmapTestOnly() { + unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL, &Data); + unmap(reinterpret_cast<void *>(RegionInfoArray), + sizeof(RegionInfo) * NumClasses); + } + + TransferBatch *popBatch(CacheT *C, uptr ClassId) { + DCHECK_LT(ClassId, NumClasses); + RegionInfo *Region = getRegionInfo(ClassId); + ScopedLock L(Region->Mutex); + TransferBatch *B = Region->FreeList.front(); + if (B) + Region->FreeList.pop_front(); + else { + B = populateFreeList(C, ClassId, Region); + if (UNLIKELY(!B)) + return nullptr; + } + DCHECK_GT(B->getCount(), 0); + Region->Stats.PoppedBlocks += B->getCount(); + return B; + } + + void pushBatch(uptr ClassId, TransferBatch *B) { + DCHECK_GT(B->getCount(), 0); + RegionInfo *Region = getRegionInfo(ClassId); + ScopedLock L(Region->Mutex); + Region->FreeList.push_front(B); + Region->Stats.PushedBlocks += B->getCount(); + if (Region->CanRelease) + releaseToOSMaybe(Region, ClassId); + } + + void disable() { + for (uptr I = 0; I < NumClasses; I++) + getRegionInfo(I)->Mutex.lock(); + } + + void enable() { + for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) + getRegionInfo(I)->Mutex.unlock(); + } + + template <typename F> void iterateOverBlocks(F Callback) const { + for (uptr I = 1; I < NumClasses; I++) { + const RegionInfo *Region = getRegionInfo(I); + const uptr BlockSize = getSizeByClassId(I); + const uptr From = Region->RegionBeg; + const uptr To = From + Region->AllocatedUser; + for (uptr Block = From; Block < To; Block += BlockSize) + Callback(Block); + } + } + + void printStats() const { + // TODO(kostyak): get the RSS per region. + uptr TotalMapped = 0; + uptr PoppedBlocks = 0; + uptr PushedBlocks = 0; + for (uptr I = 0; I < NumClasses; I++) { + RegionInfo *Region = getRegionInfo(I); + if (Region->MappedUser) + TotalMapped += Region->MappedUser; + PoppedBlocks += Region->Stats.PoppedBlocks; + PushedBlocks += Region->Stats.PushedBlocks; + } + Printf("Stats: Primary64: %zuM mapped (%zuM rss) in %zu allocations; " + "remains %zu\n", + TotalMapped >> 20, 0, PoppedBlocks, PoppedBlocks - PushedBlocks); + + for (uptr I = 0; I < NumClasses; I++) + printStats(I, 0); + } + + void releaseToOS() { + for (uptr I = 0; I < NumClasses; I++) { + if (I == SizeClassMap::BatchClassId) + continue; + RegionInfo *Region = getRegionInfo(I); + ScopedLock L(Region->Mutex); + releaseToOSMaybe(Region, I, /*Force=*/true); + } + } + +private: + static const uptr RegionSize = 1UL << RegionSizeLog; + static const uptr NumClasses = SizeClassMap::NumClasses; + static const uptr PrimarySize = RegionSize * NumClasses; + + // Call map for user memory with at least this size. + static const uptr MapSizeIncrement = 1UL << 16; + + struct RegionStats { + uptr PoppedBlocks; + uptr PushedBlocks; + }; + + struct ReleaseToOsInfo { + uptr PushedBlocksAtLastRelease; + uptr RangesReleased; + uptr LastReleasedBytes; + u64 LastReleaseAtNs; + }; + + struct ALIGNED(SCUDO_CACHE_LINE_SIZE) RegionInfo { + HybridMutex Mutex; + IntrusiveList<TransferBatch> FreeList; + RegionStats Stats; + bool CanRelease; + bool Exhausted; + u32 RandState; + uptr RegionBeg; + uptr MappedUser; // Bytes mapped for user memory. + uptr AllocatedUser; // Bytes allocated for user memory. + MapPlatformData Data; + ReleaseToOsInfo ReleaseInfo; + }; + COMPILER_CHECK(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0); + + uptr PrimaryBase; + RegionInfo *RegionInfoArray; + MapPlatformData Data; + s32 ReleaseToOsIntervalMs; + + RegionInfo *getRegionInfo(uptr ClassId) const { + DCHECK_LT(ClassId, NumClasses); + return &RegionInfoArray[ClassId]; + } + + uptr getRegionBaseByClassId(uptr ClassId) const { + return PrimaryBase + (ClassId << RegionSizeLog); + } + + bool populateBatches(CacheT *C, RegionInfo *Region, uptr ClassId, + TransferBatch **CurrentBatch, u32 MaxCount, + void **PointersArray, u32 Count) { + // No need to shuffle the batches size class. + if (ClassId != SizeClassMap::BatchClassId) + shuffle(PointersArray, Count, &Region->RandState); + TransferBatch *B = *CurrentBatch; + for (uptr I = 0; I < Count; I++) { + if (B && B->getCount() == MaxCount) { + Region->FreeList.push_back(B); + B = nullptr; + } + if (!B) { + B = C->createBatch(ClassId, PointersArray[I]); + if (UNLIKELY(!B)) + return false; + B->clear(); + } + B->add(PointersArray[I]); + } + *CurrentBatch = B; + return true; + } + + NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, + RegionInfo *Region) { + const uptr Size = getSizeByClassId(ClassId); + const u32 MaxCount = TransferBatch::getMaxCached(Size); + + const uptr RegionBeg = Region->RegionBeg; + const uptr MappedUser = Region->MappedUser; + const uptr TotalUserBytes = Region->AllocatedUser + MaxCount * Size; + // Map more space for blocks, if necessary. + if (LIKELY(TotalUserBytes > MappedUser)) { + // Do the mmap for the user memory. + const uptr UserMapSize = + roundUpTo(TotalUserBytes - MappedUser, MapSizeIncrement); + const uptr RegionBase = RegionBeg - getRegionBaseByClassId(ClassId); + if (UNLIKELY(RegionBase + MappedUser + UserMapSize > RegionSize)) { + if (!Region->Exhausted) { + Region->Exhausted = true; + printStats(); + Printf( + "Scudo OOM: The process has Exhausted %zuM for size class %zu.\n", + RegionSize >> 20, Size); + } + return nullptr; + } + if (MappedUser == 0) + Region->Data = Data; + if (UNLIKELY(!map(reinterpret_cast<void *>(RegionBeg + MappedUser), + UserMapSize, "scudo:primary", + MAP_ALLOWNOMEM | MAP_RESIZABLE, &Region->Data))) + return nullptr; + Region->MappedUser += UserMapSize; + C->getStats().add(StatMapped, UserMapSize); + } + + const uptr NumberOfBlocks = Min( + 8UL * MaxCount, (Region->MappedUser - Region->AllocatedUser) / Size); + DCHECK_GT(NumberOfBlocks, 0); + + TransferBatch *B = nullptr; + constexpr uptr ShuffleArraySize = 48; + void *ShuffleArray[ShuffleArraySize]; + u32 Count = 0; + const uptr P = RegionBeg + Region->AllocatedUser; + const uptr AllocatedUser = NumberOfBlocks * Size; + for (uptr I = P; I < P + AllocatedUser; I += Size) { + ShuffleArray[Count++] = reinterpret_cast<void *>(I); + if (Count == ShuffleArraySize) { + if (UNLIKELY(!populateBatches(C, Region, ClassId, &B, MaxCount, + ShuffleArray, Count))) + return nullptr; + Count = 0; + } + } + if (Count) { + if (UNLIKELY(!populateBatches(C, Region, ClassId, &B, MaxCount, + ShuffleArray, Count))) + return nullptr; + } + DCHECK(B); + CHECK_GT(B->getCount(), 0); + + Region->AllocatedUser += AllocatedUser; + Region->Exhausted = false; + if (Region->CanRelease) + Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTime(); + + return B; + } + + void printStats(uptr ClassId, uptr Rss) const { + RegionInfo *Region = getRegionInfo(ClassId); + if (Region->MappedUser == 0) + return; + const uptr InUse = Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks; + const uptr AvailableChunks = + Region->AllocatedUser / getSizeByClassId(ClassId); + Printf("%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu inuse: " + "%6zu avail: %6zu rss: %6zuK releases: %6zu last released: %6zuK " + "region: 0x%zx (0x%zx)\n", + Region->Exhausted ? "F" : " ", ClassId, getSizeByClassId(ClassId), + Region->MappedUser >> 10, Region->Stats.PoppedBlocks, + Region->Stats.PushedBlocks, InUse, AvailableChunks, Rss >> 10, + Region->ReleaseInfo.RangesReleased, + Region->ReleaseInfo.LastReleasedBytes >> 10, Region->RegionBeg, + getRegionBaseByClassId(ClassId)); + } + + NOINLINE void releaseToOSMaybe(RegionInfo *Region, uptr ClassId, + bool Force = false) { + const uptr BlockSize = getSizeByClassId(ClassId); + const uptr PageSize = getPageSizeCached(); + + CHECK_GE(Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks); + const uptr N = Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks; + if (N * BlockSize < PageSize) + return; // No chance to release anything. + if ((Region->Stats.PushedBlocks - + Region->ReleaseInfo.PushedBlocksAtLastRelease) * + BlockSize < + PageSize) { + return; // Nothing new to release. + } + + if (!Force) { + const s32 IntervalMs = ReleaseToOsIntervalMs; + if (IntervalMs < 0) + return; + if (Region->ReleaseInfo.LastReleaseAtNs + IntervalMs * 1000000ULL > + getMonotonicTime()) { + return; // Memory was returned recently. + } + } + + ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data); + releaseFreeMemoryToOS(&Region->FreeList, Region->RegionBeg, + roundUpTo(Region->AllocatedUser, PageSize) / PageSize, + BlockSize, &Recorder); + + if (Recorder.getReleasedRangesCount() > 0) { + Region->ReleaseInfo.PushedBlocksAtLastRelease = + Region->Stats.PushedBlocks; + Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); + Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); + } + Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTime(); + } +}; + +} // namespace scudo + +#endif // SCUDO_PRIMARY64_H_ diff --git a/lib/scudo/standalone/quarantine.h b/lib/scudo/standalone/quarantine.h new file mode 100644 index 000000000000..bac36e01c1dd --- /dev/null +++ b/lib/scudo/standalone/quarantine.h @@ -0,0 +1,289 @@ +//===-- quarantine.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_QUARANTINE_H_ +#define SCUDO_QUARANTINE_H_ + +#include "list.h" +#include "mutex.h" +#include "string_utils.h" + +namespace scudo { + +struct QuarantineBatch { + // With the following count, a batch (and the header that protects it) occupy + // 4096 bytes on 32-bit platforms, and 8192 bytes on 64-bit. + static const u32 MaxCount = 1019; + QuarantineBatch *Next; + uptr Size; + u32 Count; + void *Batch[MaxCount]; + + void init(void *Ptr, uptr Size) { + Count = 1; + Batch[0] = Ptr; + this->Size = Size + sizeof(QuarantineBatch); // Account for the Batch Size. + } + + // The total size of quarantined nodes recorded in this batch. + uptr getQuarantinedSize() const { return Size - sizeof(QuarantineBatch); } + + void push_back(void *Ptr, uptr Size) { + DCHECK_LT(Count, MaxCount); + Batch[Count++] = Ptr; + this->Size += Size; + } + + bool canMerge(const QuarantineBatch *const From) const { + return Count + From->Count <= MaxCount; + } + + void merge(QuarantineBatch *const From) { + DCHECK_LE(Count + From->Count, MaxCount); + DCHECK_GE(Size, sizeof(QuarantineBatch)); + + for (uptr I = 0; I < From->Count; ++I) + Batch[Count + I] = From->Batch[I]; + Count += From->Count; + Size += From->getQuarantinedSize(); + + From->Count = 0; + From->Size = sizeof(QuarantineBatch); + } + + void shuffle(u32 State) { ::scudo::shuffle(Batch, Count, &State); } +}; + +COMPILER_CHECK(sizeof(QuarantineBatch) <= (1U << 13)); // 8Kb. + +// Per-thread cache of memory blocks. +template <typename Callback> class QuarantineCache { +public: + void initLinkerInitialized() {} + void init() { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(); + } + + // Total memory used, including internal accounting. + uptr getSize() const { return atomic_load_relaxed(&Size); } + // Memory used for internal accounting. + uptr getOverheadSize() const { return List.size() * sizeof(QuarantineBatch); } + + void enqueue(Callback Cb, void *Ptr, uptr Size) { + if (List.empty() || List.back()->Count == QuarantineBatch::MaxCount) { + QuarantineBatch *B = + reinterpret_cast<QuarantineBatch *>(Cb.allocate(sizeof(*B))); + DCHECK(B); + B->init(Ptr, Size); + enqueueBatch(B); + } else { + List.back()->push_back(Ptr, Size); + addToSize(Size); + } + } + + void transfer(QuarantineCache *From) { + List.append_back(&From->List); + addToSize(From->getSize()); + atomic_store_relaxed(&From->Size, 0); + } + + void enqueueBatch(QuarantineBatch *B) { + List.push_back(B); + addToSize(B->Size); + } + + QuarantineBatch *dequeueBatch() { + if (List.empty()) + return nullptr; + QuarantineBatch *B = List.front(); + List.pop_front(); + subFromSize(B->Size); + return B; + } + + void mergeBatches(QuarantineCache *ToDeallocate) { + uptr ExtractedSize = 0; + QuarantineBatch *Current = List.front(); + while (Current && Current->Next) { + if (Current->canMerge(Current->Next)) { + QuarantineBatch *Extracted = Current->Next; + // Move all the chunks into the current batch. + Current->merge(Extracted); + DCHECK_EQ(Extracted->Count, 0); + DCHECK_EQ(Extracted->Size, sizeof(QuarantineBatch)); + // Remove the next batch From the list and account for its Size. + List.extract(Current, Extracted); + ExtractedSize += Extracted->Size; + // Add it to deallocation list. + ToDeallocate->enqueueBatch(Extracted); + } else { + Current = Current->Next; + } + } + subFromSize(ExtractedSize); + } + + void printStats() const { + uptr BatchCount = 0; + uptr TotalOverheadBytes = 0; + uptr TotalBytes = 0; + uptr TotalQuarantineChunks = 0; + for (const QuarantineBatch &Batch : List) { + BatchCount++; + TotalBytes += Batch.Size; + TotalOverheadBytes += Batch.Size - Batch.getQuarantinedSize(); + TotalQuarantineChunks += Batch.Count; + } + const uptr QuarantineChunksCapacity = + BatchCount * QuarantineBatch::MaxCount; + const uptr ChunksUsagePercent = + (QuarantineChunksCapacity == 0) + ? 0 + : TotalQuarantineChunks * 100 / QuarantineChunksCapacity; + const uptr TotalQuarantinedBytes = TotalBytes - TotalOverheadBytes; + const uptr MemoryOverheadPercent = + (TotalQuarantinedBytes == 0) + ? 0 + : TotalOverheadBytes * 100 / TotalQuarantinedBytes; + Printf("Global quarantine stats: batches: %zd; bytes: %zd (user: %zd); " + "chunks: %zd (capacity: %zd); %zd%% chunks used; %zd%% memory " + "overhead\n", + BatchCount, TotalBytes, TotalQuarantinedBytes, TotalQuarantineChunks, + QuarantineChunksCapacity, ChunksUsagePercent, MemoryOverheadPercent); + } + +private: + IntrusiveList<QuarantineBatch> List; + atomic_uptr Size; + + void addToSize(uptr add) { atomic_store_relaxed(&Size, getSize() + add); } + void subFromSize(uptr sub) { atomic_store_relaxed(&Size, getSize() - sub); } +}; + +// The callback interface is: +// void Callback::recycle(Node *Ptr); +// void *Callback::allocate(uptr Size); +// void Callback::deallocate(void *Ptr); +template <typename Callback, typename Node> class GlobalQuarantine { +public: + typedef QuarantineCache<Callback> CacheT; + + void initLinkerInitialized(uptr Size, uptr CacheSize) { + // Thread local quarantine size can be zero only when global quarantine size + // is zero (it allows us to perform just one atomic read per put() call). + CHECK((Size == 0 && CacheSize == 0) || CacheSize != 0); + + atomic_store_relaxed(&MaxSize, Size); + atomic_store_relaxed(&MinSize, Size / 10 * 9); // 90% of max size. + atomic_store_relaxed(&MaxCacheSize, CacheSize); + + Cache.initLinkerInitialized(); + } + void init(uptr Size, uptr CacheSize) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(Size, CacheSize); + } + + uptr getMaxSize() const { return atomic_load_relaxed(&MaxSize); } + uptr getCacheSize() const { return atomic_load_relaxed(&MaxCacheSize); } + + void put(CacheT *C, Callback Cb, Node *Ptr, uptr Size) { + C->enqueue(Cb, Ptr, Size); + if (C->getSize() > getCacheSize()) + drain(C, Cb); + } + + void NOINLINE drain(CacheT *C, Callback Cb) { + { + ScopedLock L(CacheMutex); + Cache.transfer(C); + } + if (Cache.getSize() > getMaxSize() && RecyleMutex.tryLock()) + recycle(atomic_load_relaxed(&MinSize), Cb); + } + + void NOINLINE drainAndRecycle(CacheT *C, Callback Cb) { + { + ScopedLock L(CacheMutex); + Cache.transfer(C); + } + RecyleMutex.lock(); + recycle(0, Cb); + } + + void printStats() const { + // It assumes that the world is stopped, just as the allocator's printStats. + Printf("Quarantine limits: global: %zdM; thread local: %zdK\n", + getMaxSize() >> 20, getCacheSize() >> 10); + Cache.printStats(); + } + +private: + // Read-only data. + alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex CacheMutex; + CacheT Cache; + alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex RecyleMutex; + atomic_uptr MinSize; + atomic_uptr MaxSize; + alignas(SCUDO_CACHE_LINE_SIZE) atomic_uptr MaxCacheSize; + + void NOINLINE recycle(uptr MinSize, Callback Cb) { + CacheT Tmp; + Tmp.init(); + { + ScopedLock L(CacheMutex); + // Go over the batches and merge partially filled ones to + // save some memory, otherwise batches themselves (since the memory used + // by them is counted against quarantine limit) can overcome the actual + // user's quarantined chunks, which diminishes the purpose of the + // quarantine. + const uptr CacheSize = Cache.getSize(); + const uptr OverheadSize = Cache.getOverheadSize(); + DCHECK_GE(CacheSize, OverheadSize); + // Do the merge only when overhead exceeds this predefined limit (might + // require some tuning). It saves us merge attempt when the batch list + // quarantine is unlikely to contain batches suitable for merge. + constexpr uptr OverheadThresholdPercents = 100; + if (CacheSize > OverheadSize && + OverheadSize * (100 + OverheadThresholdPercents) > + CacheSize * OverheadThresholdPercents) { + Cache.mergeBatches(&Tmp); + } + // Extract enough chunks from the quarantine to get below the max + // quarantine size and leave some leeway for the newly quarantined chunks. + while (Cache.getSize() > MinSize) + Tmp.enqueueBatch(Cache.dequeueBatch()); + } + RecyleMutex.unlock(); + doRecycle(&Tmp, Cb); + } + + void NOINLINE doRecycle(CacheT *C, Callback Cb) { + while (QuarantineBatch *B = C->dequeueBatch()) { + const u32 Seed = static_cast<u32>( + (reinterpret_cast<uptr>(B) ^ reinterpret_cast<uptr>(C)) >> 4); + B->shuffle(Seed); + constexpr uptr NumberOfPrefetch = 8UL; + CHECK(NumberOfPrefetch <= ARRAY_SIZE(B->Batch)); + for (uptr I = 0; I < NumberOfPrefetch; I++) + PREFETCH(B->Batch[I]); + for (uptr I = 0, Count = B->Count; I < Count; I++) { + if (I + NumberOfPrefetch < Count) + PREFETCH(B->Batch[I + NumberOfPrefetch]); + Cb.recycle(reinterpret_cast<Node *>(B->Batch[I])); + } + Cb.deallocate(B); + } + } +}; + +} // namespace scudo + +#endif // SCUDO_QUARANTINE_H_ diff --git a/lib/scudo/standalone/release.h b/lib/scudo/standalone/release.h new file mode 100644 index 000000000000..4fe29fde4bde --- /dev/null +++ b/lib/scudo/standalone/release.h @@ -0,0 +1,262 @@ +//===-- release.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_RELEASE_H_ +#define SCUDO_RELEASE_H_ + +#include "common.h" +#include "list.h" + +namespace scudo { + +class ReleaseRecorder { +public: + ReleaseRecorder(uptr BaseAddress, MapPlatformData *Data = nullptr) + : BaseAddress(BaseAddress), Data(Data) {} + + uptr getReleasedRangesCount() const { return ReleasedRangesCount; } + + uptr getReleasedBytes() const { return ReleasedBytes; } + + // Releases [From, To) range of pages back to OS. + void releasePageRangeToOS(uptr From, uptr To) { + const uptr Size = To - From; + releasePagesToOS(BaseAddress, From, Size, Data); + ReleasedRangesCount++; + ReleasedBytes += Size; + } + +private: + uptr ReleasedRangesCount = 0; + uptr ReleasedBytes = 0; + uptr BaseAddress = 0; + MapPlatformData *Data = nullptr; +}; + +// A packed array of Counters. Each counter occupies 2^N bits, enough to store +// counter's MaxValue. Ctor will try to allocate the required Buffer via map() +// and the caller is expected to check whether the initialization was successful +// by checking isAllocated() result. For the performance sake, none of the +// accessors check the validity of the arguments, It is assumed that Index is +// always in [0, N) range and the value is not incremented past MaxValue. +class PackedCounterArray { +public: + PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) { + CHECK_GT(NumCounters, 0); + CHECK_GT(MaxValue, 0); + constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL; + // Rounding counter storage size up to the power of two allows for using + // bit shifts calculating particular counter's Index and offset. + const uptr CounterSizeBits = + roundUpToPowerOfTwo(getMostSignificantSetBitIndex(MaxValue) + 1); + CHECK_LE(CounterSizeBits, MaxCounterBits); + CounterSizeBitsLog = getLog2(CounterSizeBits); + CounterMask = ~(static_cast<uptr>(0)) >> (MaxCounterBits - CounterSizeBits); + + const uptr PackingRatio = MaxCounterBits >> CounterSizeBitsLog; + CHECK_GT(PackingRatio, 0); + PackingRatioLog = getLog2(PackingRatio); + BitOffsetMask = PackingRatio - 1; + + BufferSize = (roundUpTo(N, static_cast<uptr>(1U) << PackingRatioLog) >> + PackingRatioLog) * + sizeof(*Buffer); + Buffer = reinterpret_cast<uptr *>( + map(nullptr, BufferSize, "scudo:counters", MAP_ALLOWNOMEM)); + } + ~PackedCounterArray() { + if (isAllocated()) + unmap(reinterpret_cast<void *>(Buffer), BufferSize); + } + + bool isAllocated() const { return !!Buffer; } + + uptr getCount() const { return N; } + + uptr get(uptr I) const { + DCHECK_LT(I, N); + const uptr Index = I >> PackingRatioLog; + const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; + return (Buffer[Index] >> BitOffset) & CounterMask; + } + + void inc(uptr I) const { + DCHECK_LT(get(I), CounterMask); + const uptr Index = I >> PackingRatioLog; + const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; + DCHECK_LT(BitOffset, SCUDO_WORDSIZE); + Buffer[Index] += static_cast<uptr>(1U) << BitOffset; + } + + void incRange(uptr From, uptr To) const { + DCHECK_LE(From, To); + for (uptr I = From; I <= To; I++) + inc(I); + } + + uptr getBufferSize() const { return BufferSize; } + +private: + const uptr N; + uptr CounterSizeBitsLog; + uptr CounterMask; + uptr PackingRatioLog; + uptr BitOffsetMask; + + uptr BufferSize; + uptr *Buffer; +}; + +template <class ReleaseRecorderT> class FreePagesRangeTracker { +public: + explicit FreePagesRangeTracker(ReleaseRecorderT *Recorder) + : Recorder(Recorder), PageSizeLog(getLog2(getPageSizeCached())) {} + + void processNextPage(bool Freed) { + if (Freed) { + if (!InRange) { + CurrentRangeStatePage = CurrentPage; + InRange = true; + } + } else { + closeOpenedRange(); + } + CurrentPage++; + } + + void finish() { closeOpenedRange(); } + +private: + void closeOpenedRange() { + if (InRange) { + Recorder->releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog), + (CurrentPage << PageSizeLog)); + InRange = false; + } + } + + ReleaseRecorderT *const Recorder; + const uptr PageSizeLog; + bool InRange = false; + uptr CurrentPage = 0; + uptr CurrentRangeStatePage = 0; +}; + +template <class TransferBatchT, class ReleaseRecorderT> +NOINLINE void +releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> *FreeList, uptr Base, + uptr AllocatedPagesCount, uptr BlockSize, + ReleaseRecorderT *Recorder) { + const uptr PageSize = getPageSizeCached(); + + // Figure out the number of chunks per page and whether we can take a fast + // path (the number of chunks per page is the same for all pages). + uptr FullPagesBlockCountMax; + bool SameBlockCountPerPage; + if (BlockSize <= PageSize) { + if (PageSize % BlockSize == 0) { + // Same number of chunks per page, no cross overs. + FullPagesBlockCountMax = PageSize / BlockSize; + SameBlockCountPerPage = true; + } else if (BlockSize % (PageSize % BlockSize) == 0) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks, but all pages contain the same + // number of chunks. + FullPagesBlockCountMax = PageSize / BlockSize + 1; + SameBlockCountPerPage = true; + } else { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks. + FullPagesBlockCountMax = PageSize / BlockSize + 2; + SameBlockCountPerPage = false; + } + } else { + if (BlockSize % PageSize == 0) { + // One chunk covers multiple pages, no cross overs. + FullPagesBlockCountMax = 1; + SameBlockCountPerPage = true; + } else { + // One chunk covers multiple pages, Some chunks are crossing page + // boundaries. Some pages contain one chunk, some contain two. + FullPagesBlockCountMax = 2; + SameBlockCountPerPage = false; + } + } + + PackedCounterArray Counters(AllocatedPagesCount, FullPagesBlockCountMax); + if (!Counters.isAllocated()) + return; + + const uptr PageSizeLog = getLog2(PageSize); + const uptr End = Base + AllocatedPagesCount * PageSize; + + // Iterate over free chunks and count how many free chunks affect each + // allocated page. + if (BlockSize <= PageSize && PageSize % BlockSize == 0) { + // Each chunk affects one page only. + for (auto It = FreeList->begin(); It != FreeList->end(); ++It) { + for (u32 I = 0; I < (*It).getCount(); I++) { + const uptr P = reinterpret_cast<uptr>((*It).get(I)); + if (P >= Base && P < End) + Counters.inc((P - Base) >> PageSizeLog); + } + } + } else { + // In all other cases chunks might affect more than one page. + for (auto It = FreeList->begin(); It != FreeList->end(); ++It) { + for (u32 I = 0; I < (*It).getCount(); I++) { + const uptr P = reinterpret_cast<uptr>((*It).get(I)); + if (P >= Base && P < End) + Counters.incRange((P - Base) >> PageSizeLog, + (P - Base + BlockSize - 1) >> PageSizeLog); + } + } + } + + // Iterate over pages detecting ranges of pages with chunk Counters equal + // to the expected number of chunks for the particular page. + FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder); + if (SameBlockCountPerPage) { + // Fast path, every page has the same number of chunks affecting it. + for (uptr I = 0; I < Counters.getCount(); I++) + RangeTracker.processNextPage(Counters.get(I) == FullPagesBlockCountMax); + } else { + // Slow path, go through the pages keeping count how many chunks affect + // each page. + const uptr Pn = BlockSize < PageSize ? PageSize / BlockSize : 1; + const uptr Pnc = Pn * BlockSize; + // The idea is to increment the current page pointer by the first chunk + // size, middle portion size (the portion of the page covered by chunks + // except the first and the last one) and then the last chunk size, adding + // up the number of chunks on the current page and checking on every step + // whether the page boundary was crossed. + uptr PrevPageBoundary = 0; + uptr CurrentBoundary = 0; + for (uptr I = 0; I < Counters.getCount(); I++) { + const uptr PageBoundary = PrevPageBoundary + PageSize; + uptr BlocksPerPage = Pn; + if (CurrentBoundary < PageBoundary) { + if (CurrentBoundary > PrevPageBoundary) + BlocksPerPage++; + CurrentBoundary += Pnc; + if (CurrentBoundary < PageBoundary) { + BlocksPerPage++; + CurrentBoundary += BlockSize; + } + } + PrevPageBoundary = PageBoundary; + + RangeTracker.processNextPage(Counters.get(I) == BlocksPerPage); + } + } + RangeTracker.finish(); +} + +} // namespace scudo + +#endif // SCUDO_RELEASE_H_ diff --git a/lib/scudo/standalone/report.cc b/lib/scudo/standalone/report.cc new file mode 100644 index 000000000000..47cd951e8ed4 --- /dev/null +++ b/lib/scudo/standalone/report.cc @@ -0,0 +1,192 @@ +//===-- report.cc -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "report.h" + +#include "atomic_helpers.h" +#include "string_utils.h" + +#include <stdarg.h> + +namespace scudo { + +class ScopedErrorReport { +public: + ScopedErrorReport() : Message(512) { Message.append("Scudo ERROR: "); } + void append(const char *Format, ...) { + va_list Args; + va_start(Args, Format); + Message.append(Format, Args); + va_end(Args); + } + NORETURN ~ScopedErrorReport() { + outputRaw(Message.data()); + setAbortMessage(Message.data()); + die(); + } + +private: + ScopedString Message; +}; + +INLINE void NORETURN trap() { __builtin_trap(); } + +// This could potentially be called recursively if a CHECK fails in the reports. +void NORETURN reportCheckFailed(const char *File, int Line, + const char *Condition, u64 Value1, u64 Value2) { + static atomic_u32 NumberOfCalls; + if (atomic_fetch_add(&NumberOfCalls, 1, memory_order_relaxed) > 2) { + // TODO(kostyak): maybe sleep here? + trap(); + } + ScopedErrorReport Report; + Report.append("CHECK failed @ %s:%d %s (%llu, %llu)\n", File, Line, Condition, + Value1, Value2); +} + +// Generic string fatal error message. +void NORETURN reportError(const char *Message) { + ScopedErrorReport Report; + Report.append("%s\n", Message); +} + +void NORETURN reportInvalidFlag(const char *FlagType, const char *Value) { + ScopedErrorReport Report; + Report.append("invalid value for %s option: '%s'\n", FlagType, Value); +} + +// The checksum of a chunk header is invalid. This could be caused by an +// {over,under}write of the header, a pointer that is not an actual chunk. +void NORETURN reportHeaderCorruption(void *Ptr) { + ScopedErrorReport Report; + Report.append("corrupted chunk header at address %p\n", Ptr); +} + +// Two threads have attempted to modify a chunk header at the same time. This is +// symptomatic of a race-condition in the application code, or general lack of +// proper locking. +void NORETURN reportHeaderRace(void *Ptr) { + ScopedErrorReport Report; + Report.append("race on chunk header at address %p\n", Ptr); +} + +// The allocator was compiled with parameters that conflict with field size +// requirements. +void NORETURN reportSanityCheckError(const char *Field) { + ScopedErrorReport Report; + Report.append("maximum possible %s doesn't fit in header\n", Field); +} + +// We enforce a maximum alignment, to keep fields smaller and generally prevent +// integer overflows, or unexpected corner cases. +void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment) { + ScopedErrorReport Report; + Report.append("invalid allocation alignment: %zu exceeds maximum supported " + "alignment of %zu\n", + Alignment, MaxAlignment); +} + +// See above, we also enforce a maximum size. +void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, + uptr MaxSize) { + ScopedErrorReport Report; + Report.append("requested allocation size %zu (%zu after adjustments) exceeds " + "maximum supported size of %zu\n", + UserSize, TotalSize, MaxSize); +} + +void NORETURN reportOutOfMemory(uptr RequestedSize) { + ScopedErrorReport Report; + Report.append("out of memory trying to allocate %zu bytes\n", RequestedSize); +} + +static const char *stringifyAction(AllocatorAction Action) { + switch (Action) { + case AllocatorAction::Recycling: + return "recycling"; + case AllocatorAction::Deallocating: + return "deallocating"; + case AllocatorAction::Reallocating: + return "reallocating"; + case AllocatorAction::Sizing: + return "sizing"; + } + return "<invalid action>"; +} + +// The chunk is not in a state congruent with the operation we want to perform. +// This is usually the case with a double-free, a realloc of a freed pointer. +void NORETURN reportInvalidChunkState(AllocatorAction Action, void *Ptr) { + ScopedErrorReport Report; + Report.append("invalid chunk state when %s address %p\n", + stringifyAction(Action), Ptr); +} + +void NORETURN reportMisalignedPointer(AllocatorAction Action, void *Ptr) { + ScopedErrorReport Report; + Report.append("misaligned pointer when %s address %p\n", + stringifyAction(Action), Ptr); +} + +// The deallocation function used is at odds with the one used to allocate the +// chunk (eg: new[]/delete or malloc/delete, and so on). +void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, void *Ptr, + u8 TypeA, u8 TypeB) { + ScopedErrorReport Report; + Report.append("allocation type mismatch when %s address %p (%d vs %d)\n", + stringifyAction(Action), Ptr, TypeA, TypeB); +} + +// The size specified to the delete operator does not match the one that was +// passed to new when allocating the chunk. +void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size, + uptr ExpectedSize) { + ScopedErrorReport Report; + Report.append( + "invalid sized delete when deallocating address %p (%zu vs %zu)\n", Ptr, + Size, ExpectedSize); +} + +void NORETURN reportAlignmentNotPowerOfTwo(uptr Alignment) { + ScopedErrorReport Report; + Report.append( + "invalid allocation alignment: %zu, alignment must be a power of two\n", + Alignment); +} + +void NORETURN reportCallocOverflow(uptr Count, uptr Size) { + ScopedErrorReport Report; + Report.append("calloc parameters overflow: count * size (%zu * %zu) cannot " + "be represented with type size_t\n", + Count, Size); +} + +void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment) { + ScopedErrorReport Report; + Report.append( + "invalid alignment requested in posix_memalign: %zu, alignment must be a " + "power of two and a multiple of sizeof(void *) == %zu\n", + Alignment, sizeof(void *)); +} + +void NORETURN reportPvallocOverflow(uptr Size) { + ScopedErrorReport Report; + Report.append("pvalloc parameters overflow: size %zu rounded up to system " + "page size %zu cannot be represented in type size_t\n", + Size, getPageSizeCached()); +} + +void NORETURN reportInvalidAlignedAllocAlignment(uptr Alignment, uptr Size) { + ScopedErrorReport Report; + Report.append("invalid alignment requested in aligned_alloc: %zu, alignment " + "must be a power of two and the requested size %zu must be a " + "multiple of alignment\n", + Alignment, Size); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/report.h b/lib/scudo/standalone/report.h new file mode 100644 index 000000000000..14e4e799b736 --- /dev/null +++ b/lib/scudo/standalone/report.h @@ -0,0 +1,57 @@ +//===-- report.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_REPORT_H_ +#define SCUDO_REPORT_H_ + +#include "internal_defs.h" + +namespace scudo { + +// Reports are *fatal* unless stated otherwise. + +// Generic error. +void NORETURN reportError(const char *Message); + +// Flags related errors. +void NORETURN reportInvalidFlag(const char *FlagType, const char *Value); + +// Chunk header related errors. +void NORETURN reportHeaderCorruption(void *Ptr); +void NORETURN reportHeaderRace(void *Ptr); + +// Sanity checks related error. +void NORETURN reportSanityCheckError(const char *Field); + +// Combined allocator errors. +void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment); +void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, + uptr MaxSize); +void NORETURN reportOutOfMemory(uptr RequestedSize); +enum class AllocatorAction : u8 { + Recycling, + Deallocating, + Reallocating, + Sizing, +}; +void NORETURN reportInvalidChunkState(AllocatorAction Action, void *Ptr); +void NORETURN reportMisalignedPointer(AllocatorAction Action, void *Ptr); +void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, void *Ptr, + u8 TypeA, u8 TypeB); +void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size, uptr ExpectedSize); + +// C wrappers errors. +void NORETURN reportAlignmentNotPowerOfTwo(uptr Alignment); +void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment); +void NORETURN reportCallocOverflow(uptr Count, uptr Size); +void NORETURN reportPvallocOverflow(uptr Size); +void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment); + +} // namespace scudo + +#endif // SCUDO_REPORT_H_ diff --git a/lib/scudo/standalone/secondary.cc b/lib/scudo/standalone/secondary.cc new file mode 100644 index 000000000000..75f9171f1617 --- /dev/null +++ b/lib/scudo/standalone/secondary.cc @@ -0,0 +1,136 @@ +//===-- secondary.cc --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "secondary.h" + +#include "string_utils.h" + +namespace scudo { + +// As with the Primary, the size passed to this function includes any desired +// alignment, so that the frontend can align the user allocation. The hint +// parameter allows us to unmap spurious memory when dealing with larger +// (greater than a page) alignments on 32-bit platforms. +// Due to the sparsity of address space available on those platforms, requesting +// an allocation from the Secondary with a large alignment would end up wasting +// VA space (even though we are not committing the whole thing), hence the need +// to trim off some of the reserved space. +// For allocations requested with an alignment greater than or equal to a page, +// the committed memory will amount to something close to Size - AlignmentHint +// (pending rounding and headers). +void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, uptr *BlockEnd) { + DCHECK_GT(Size, AlignmentHint); + const uptr PageSize = getPageSizeCached(); + const uptr MapSize = + roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize) + 2 * PageSize; + MapPlatformData Data = {}; + uptr MapBase = + reinterpret_cast<uptr>(map(nullptr, MapSize, "scudo:secondary", + MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); + if (!MapBase) + return nullptr; + uptr CommitBase = MapBase + PageSize; + uptr MapEnd = MapBase + MapSize; + + // In the unlikely event of alignments larger than a page, adjust the amount + // of memory we want to commit, and trim the extra memory. + if (AlignmentHint >= PageSize) { + // For alignments greater than or equal to a page, the user pointer (eg: the + // pointer that is returned by the C or C++ allocation APIs) ends up on a + // page boundary , and our headers will live in the preceding page. + CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize; + const uptr NewMapBase = CommitBase - PageSize; + DCHECK_GE(NewMapBase, MapBase); + // We only trim the extra memory on 32-bit platforms: 64-bit platforms + // are less constrained memory wise, and that saves us two syscalls. + if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) { + unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data); + MapBase = NewMapBase; + } + const uptr NewMapEnd = CommitBase + PageSize + + roundUpTo((Size - AlignmentHint), PageSize) + + PageSize; + DCHECK_LE(NewMapEnd, MapEnd); + if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) { + unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data); + MapEnd = NewMapEnd; + } + } + + const uptr CommitSize = MapEnd - PageSize - CommitBase; + const uptr Ptr = + reinterpret_cast<uptr>(map(reinterpret_cast<void *>(CommitBase), + CommitSize, "scudo:secondary", 0, &Data)); + LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(Ptr); + H->MapBase = MapBase; + H->MapSize = MapEnd - MapBase; + H->BlockEnd = CommitBase + CommitSize; + H->Data = Data; + { + ScopedLock L(Mutex); + if (!Tail) { + Tail = H; + } else { + Tail->Next = H; + H->Prev = Tail; + Tail = H; + } + AllocatedBytes += CommitSize; + if (LargestSize < CommitSize) + LargestSize = CommitSize; + NumberOfAllocs++; + Stats.add(StatAllocated, CommitSize); + Stats.add(StatMapped, H->MapSize); + } + if (BlockEnd) + *BlockEnd = CommitBase + CommitSize; + return reinterpret_cast<void *>(Ptr + LargeBlock::getHeaderSize()); +} + +void MapAllocator::deallocate(void *Ptr) { + LargeBlock::Header *H = LargeBlock::getHeader(Ptr); + { + ScopedLock L(Mutex); + LargeBlock::Header *Prev = H->Prev; + LargeBlock::Header *Next = H->Next; + if (Prev) { + CHECK_EQ(Prev->Next, H); + Prev->Next = Next; + } + if (Next) { + CHECK_EQ(Next->Prev, H); + Next->Prev = Prev; + } + if (Tail == H) { + CHECK(!Next); + Tail = Prev; + } else { + CHECK(Next); + } + const uptr CommitSize = H->BlockEnd - reinterpret_cast<uptr>(H); + FreedBytes += CommitSize; + NumberOfFrees++; + Stats.sub(StatAllocated, CommitSize); + Stats.sub(StatMapped, H->MapSize); + } + void *Addr = reinterpret_cast<void *>(H->MapBase); + const uptr Size = H->MapSize; + MapPlatformData Data; + Data = H->Data; + unmap(Addr, Size, UNMAP_ALL, &Data); +} + +void MapAllocator::printStats() const { + Printf("Stats: MapAllocator: allocated %zd times (%zdK), freed %zd times " + "(%zdK), remains %zd (%zdK) max %zdM\n", + NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, + NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10, + LargestSize >> 20); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/secondary.h b/lib/scudo/standalone/secondary.h new file mode 100644 index 000000000000..9124e2a41c6a --- /dev/null +++ b/lib/scudo/standalone/secondary.h @@ -0,0 +1,97 @@ +//===-- secondary.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_SECONDARY_H_ +#define SCUDO_SECONDARY_H_ + +#include "common.h" +#include "mutex.h" +#include "stats.h" + +namespace scudo { + +// This allocator wraps the platform allocation primitives, and as such is on +// the slower side and should preferably be used for larger sized allocations. +// Blocks allocated will be preceded and followed by a guard page, and hold +// their own header that is not checksummed: the guard pages and the Combined +// header should be enough for our purpose. + +namespace LargeBlock { + +struct Header { + LargeBlock::Header *Prev; + LargeBlock::Header *Next; + uptr BlockEnd; + uptr MapBase; + uptr MapSize; + MapPlatformData Data; +}; + +constexpr uptr getHeaderSize() { + return roundUpTo(sizeof(Header), 1U << SCUDO_MIN_ALIGNMENT_LOG); +} + +static Header *getHeader(uptr Ptr) { + return reinterpret_cast<Header *>(Ptr - getHeaderSize()); +} + +static Header *getHeader(const void *Ptr) { + return getHeader(reinterpret_cast<uptr>(Ptr)); +} + +} // namespace LargeBlock + +class MapAllocator { +public: + void initLinkerInitialized(GlobalStats *S) { + Stats.initLinkerInitialized(); + if (S) + S->link(&Stats); + } + void init(GlobalStats *S) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(S); + } + + void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr); + + void deallocate(void *Ptr); + + static uptr getBlockEnd(void *Ptr) { + return LargeBlock::getHeader(Ptr)->BlockEnd; + } + + static uptr getBlockSize(void *Ptr) { + return getBlockEnd(Ptr) - reinterpret_cast<uptr>(Ptr); + } + + void printStats() const; + + void disable() { Mutex.lock(); } + + void enable() { Mutex.unlock(); } + + template <typename F> void iterateOverBlocks(F Callback) const { + for (LargeBlock::Header *H = Tail; H != nullptr; H = H->Prev) + Callback(reinterpret_cast<uptr>(H) + LargeBlock::getHeaderSize()); + } + +private: + HybridMutex Mutex; + LargeBlock::Header *Tail; + uptr AllocatedBytes; + uptr FreedBytes; + uptr LargestSize; + u32 NumberOfAllocs; + u32 NumberOfFrees; + LocalStats Stats; +}; + +} // namespace scudo + +#endif // SCUDO_SECONDARY_H_ diff --git a/lib/scudo/standalone/size_class_map.h b/lib/scudo/standalone/size_class_map.h new file mode 100644 index 000000000000..b7df54cf8098 --- /dev/null +++ b/lib/scudo/standalone/size_class_map.h @@ -0,0 +1,149 @@ +//===-- size_class_map.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_SIZE_CLASS_MAP_H_ +#define SCUDO_SIZE_CLASS_MAP_H_ + +#include "common.h" +#include "string_utils.h" + +namespace scudo { + +// SizeClassMap maps allocation sizes into size classes and back, in an +// efficient table-free manner. +// +// Class 0 is a special class that doesn't abide by the same rules as other +// classes. The allocator uses it to hold batches. +// +// The other sizes are controlled by the template parameters: +// - MinSizeLog: defines the first class as 2^MinSizeLog bytes. +// - MaxSizeLog: defines the last class as 2^MaxSizeLog bytes. +// - MidSizeLog: classes increase with step 2^MinSizeLog from 2^MinSizeLog to +// 2^MidSizeLog bytes. +// - NumBits: the number of non-zero bits in sizes after 2^MidSizeLog. +// eg. with NumBits==3 all size classes after 2^MidSizeLog look like +// 0b1xx0..0 (where x is either 0 or 1). +// +// This class also gives a hint to a thread-caching allocator about the amount +// of chunks that can be cached per-thread: +// - MaxNumCachedHint is a hint for the max number of chunks cached per class. +// - 2^MaxBytesCachedLog is the max number of bytes cached per class. + +template <u8 NumBits, u8 MinSizeLog, u8 MidSizeLog, u8 MaxSizeLog, + u32 MaxNumCachedHintT, u8 MaxBytesCachedLog> +class SizeClassMap { + static const uptr MinSize = 1UL << MinSizeLog; + static const uptr MidSize = 1UL << MidSizeLog; + static const uptr MidClass = MidSize / MinSize; + static const u8 S = NumBits - 1; + static const uptr M = (1UL << S) - 1; + +public: + static const u32 MaxNumCachedHint = MaxNumCachedHintT; + + static const uptr MaxSize = 1UL << MaxSizeLog; + static const uptr NumClasses = + MidClass + ((MaxSizeLog - MidSizeLog) << S) + 1; + COMPILER_CHECK(NumClasses <= 256); + static const uptr LargestClassId = NumClasses - 1; + static const uptr BatchClassId = 0; + + static uptr getSizeByClassId(uptr ClassId) { + DCHECK_NE(ClassId, BatchClassId); + if (ClassId <= MidClass) + return ClassId << MinSizeLog; + ClassId -= MidClass; + const uptr T = MidSize << (ClassId >> S); + return T + (T >> S) * (ClassId & M); + } + + static uptr getClassIdBySize(uptr Size) { + DCHECK_LE(Size, MaxSize); + if (Size <= MidSize) + return (Size + MinSize - 1) >> MinSizeLog; + const uptr L = getMostSignificantSetBitIndex(Size); + const uptr HBits = (Size >> (L - S)) & M; + const uptr LBits = Size & ((1UL << (L - S)) - 1); + const uptr L1 = L - MidSizeLog; + return MidClass + (L1 << S) + HBits + (LBits > 0); + } + + static u32 getMaxCachedHint(uptr Size) { + DCHECK_LE(Size, MaxSize); + DCHECK_NE(Size, 0); + u32 N; + // Force a 32-bit division if the template parameters allow for it. + if (MaxBytesCachedLog > 31 || MaxSizeLog > 31) + N = static_cast<u32>((1UL << MaxBytesCachedLog) / Size); + else + N = (1U << MaxBytesCachedLog) / static_cast<u32>(Size); + return Max(1U, Min(MaxNumCachedHint, N)); + } + + static void print() { + uptr PrevS = 0; + uptr TotalCached = 0; + for (uptr I = 0; I < NumClasses; I++) { + if (I == BatchClassId) + continue; + const uptr S = getSizeByClassId(I); + if (S >= MidSize / 2 && (S & (S - 1)) == 0) + Printf("\n"); + const uptr D = S - PrevS; + const uptr P = PrevS ? (D * 100 / PrevS) : 0; + const uptr L = S ? getMostSignificantSetBitIndex(S) : 0; + const uptr Cached = getMaxCachedHint(S) * S; + Printf( + "C%02zu => S: %zu diff: +%zu %02zu%% L %zu Cached: %zu %zu; id %zu\n", + I, getSizeByClassId(I), D, P, L, getMaxCachedHint(S), Cached, + getClassIdBySize(S)); + TotalCached += Cached; + PrevS = S; + } + Printf("Total Cached: %zu\n", TotalCached); + } + + static void validate() { + for (uptr C = 0; C < NumClasses; C++) { + if (C == BatchClassId) + continue; + const uptr S = getSizeByClassId(C); + CHECK_NE(S, 0U); + CHECK_EQ(getClassIdBySize(S), C); + if (C < LargestClassId) + CHECK_EQ(getClassIdBySize(S + 1), C + 1); + CHECK_EQ(getClassIdBySize(S - 1), C); + CHECK_GT(getSizeByClassId(C), getSizeByClassId(C - 1)); + } + // Do not perform the loop if the maximum size is too large. + if (MaxSizeLog > 19) + return; + for (uptr S = 1; S <= MaxSize; S++) { + const uptr C = getClassIdBySize(S); + CHECK_LT(C, NumClasses); + CHECK_GE(getSizeByClassId(C), S); + if (C > 0) + CHECK_LT(getSizeByClassId(C - 1), S); + } + } +}; + +typedef SizeClassMap<3, 5, 8, 17, 8, 10> DefaultSizeClassMap; + +// TODO(kostyak): further tune class maps for Android & Fuchsia. +#if SCUDO_WORDSIZE == 64U +typedef SizeClassMap<3, 5, 8, 15, 8, 10> SvelteSizeClassMap; +typedef SizeClassMap<3, 5, 8, 16, 14, 12> AndroidSizeClassMap; +#else +typedef SizeClassMap<3, 4, 7, 15, 8, 10> SvelteSizeClassMap; +typedef SizeClassMap<3, 4, 7, 16, 14, 12> AndroidSizeClassMap; +#endif + +} // namespace scudo + +#endif // SCUDO_SIZE_CLASS_MAP_H_ diff --git a/lib/scudo/standalone/stats.h b/lib/scudo/standalone/stats.h new file mode 100644 index 000000000000..12436756226b --- /dev/null +++ b/lib/scudo/standalone/stats.h @@ -0,0 +1,105 @@ +//===-- stats.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_STATS_H_ +#define SCUDO_STATS_H_ + +#include "atomic_helpers.h" +#include "mutex.h" + +#include <string.h> + +namespace scudo { + +// Memory allocator statistics +enum StatType { StatAllocated, StatMapped, StatCount }; + +typedef uptr StatCounters[StatCount]; + +// Per-thread stats, live in per-thread cache. We use atomics so that the +// numbers themselves are consistent. But we don't use atomic_{add|sub} or a +// lock, because those are expensive operations , and we only care for the stats +// to be "somewhat" correct: eg. if we call GlobalStats::get while a thread is +// LocalStats::add'ing, this is OK, we will still get a meaningful number. +class LocalStats { +public: + void initLinkerInitialized() {} + void init() { memset(this, 0, sizeof(*this)); } + + void add(StatType I, uptr V) { + V += atomic_load_relaxed(&StatsArray[I]); + atomic_store_relaxed(&StatsArray[I], V); + } + + void sub(StatType I, uptr V) { + V = atomic_load_relaxed(&StatsArray[I]) - V; + atomic_store_relaxed(&StatsArray[I], V); + } + + void set(StatType I, uptr V) { atomic_store_relaxed(&StatsArray[I], V); } + + uptr get(StatType I) const { return atomic_load_relaxed(&StatsArray[I]); } + +private: + friend class GlobalStats; + atomic_uptr StatsArray[StatCount]; + LocalStats *Next; + LocalStats *Prev; +}; + +// Global stats, used for aggregation and querying. +class GlobalStats : public LocalStats { +public: + void initLinkerInitialized() { + Next = this; + Prev = this; + } + void init() { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(); + } + + void link(LocalStats *S) { + ScopedLock L(Mutex); + S->Next = Next; + S->Prev = this; + Next->Prev = S; + Next = S; + } + + void unlink(LocalStats *S) { + ScopedLock L(Mutex); + S->Prev->Next = S->Next; + S->Next->Prev = S->Prev; + for (uptr I = 0; I < StatCount; I++) + add(static_cast<StatType>(I), S->get(static_cast<StatType>(I))); + } + + void get(uptr *S) const { + memset(S, 0, StatCount * sizeof(uptr)); + ScopedLock L(Mutex); + const LocalStats *Stats = this; + for (;;) { + for (uptr I = 0; I < StatCount; I++) + S[I] += Stats->get(static_cast<StatType>(I)); + Stats = Stats->Next; + if (Stats == this) + break; + } + // All stats must be non-negative. + for (uptr I = 0; I < StatCount; I++) + S[I] = static_cast<sptr>(S[I]) >= 0 ? S[I] : 0; + } + +private: + mutable HybridMutex Mutex; +}; + +} // namespace scudo + +#endif // SCUDO_STATS_H_ diff --git a/lib/scudo/standalone/string_utils.cc b/lib/scudo/standalone/string_utils.cc new file mode 100644 index 000000000000..f0068afc1e8b --- /dev/null +++ b/lib/scudo/standalone/string_utils.cc @@ -0,0 +1,236 @@ +//===-- string_utils.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "string_utils.h" +#include "common.h" + +#include <ctype.h> +#include <stdarg.h> +#include <string.h> + +namespace scudo { + +static int appendChar(char **Buffer, const char *BufferEnd, char C) { + if (*Buffer < BufferEnd) { + **Buffer = C; + (*Buffer)++; + } + return 1; +} + +// Appends number in a given Base to buffer. If its length is less than +// |MinNumberLength|, it is padded with leading zeroes or spaces, depending +// on the value of |PadWithZero|. +static int appendNumber(char **Buffer, const char *BufferEnd, u64 AbsoluteValue, + u8 Base, u8 MinNumberLength, bool PadWithZero, + bool Negative, bool Upper) { + constexpr uptr MaxLen = 30; + RAW_CHECK(Base == 10 || Base == 16); + RAW_CHECK(Base == 10 || !Negative); + RAW_CHECK(AbsoluteValue || !Negative); + RAW_CHECK(MinNumberLength < MaxLen); + int Res = 0; + if (Negative && MinNumberLength) + --MinNumberLength; + if (Negative && PadWithZero) + Res += appendChar(Buffer, BufferEnd, '-'); + uptr NumBuffer[MaxLen]; + int Pos = 0; + do { + RAW_CHECK_MSG(static_cast<uptr>(Pos) < MaxLen, + "appendNumber buffer overflow"); + NumBuffer[Pos++] = AbsoluteValue % Base; + AbsoluteValue /= Base; + } while (AbsoluteValue > 0); + if (Pos < MinNumberLength) { + memset(&NumBuffer[Pos], 0, + sizeof(NumBuffer[0]) * static_cast<uptr>(MinNumberLength - Pos)); + Pos = MinNumberLength; + } + RAW_CHECK(Pos > 0); + Pos--; + for (; Pos >= 0 && NumBuffer[Pos] == 0; Pos--) { + char c = (PadWithZero || Pos == 0) ? '0' : ' '; + Res += appendChar(Buffer, BufferEnd, c); + } + if (Negative && !PadWithZero) + Res += appendChar(Buffer, BufferEnd, '-'); + for (; Pos >= 0; Pos--) { + char Digit = static_cast<char>(NumBuffer[Pos]); + Digit = static_cast<char>((Digit < 10) ? '0' + Digit + : (Upper ? 'A' : 'a') + Digit - 10); + Res += appendChar(Buffer, BufferEnd, Digit); + } + return Res; +} + +static int appendUnsigned(char **Buffer, const char *BufferEnd, u64 Num, + u8 Base, u8 MinNumberLength, bool PadWithZero, + bool Upper) { + return appendNumber(Buffer, BufferEnd, Num, Base, MinNumberLength, + PadWithZero, /*Negative=*/false, Upper); +} + +static int appendSignedDecimal(char **Buffer, const char *BufferEnd, s64 Num, + u8 MinNumberLength, bool PadWithZero) { + const bool Negative = (Num < 0); + return appendNumber(Buffer, BufferEnd, + static_cast<u64>(Negative ? -Num : Num), 10, + MinNumberLength, PadWithZero, Negative, + /*Upper=*/false); +} + +// Use the fact that explicitly requesting 0 Width (%0s) results in UB and +// interpret Width == 0 as "no Width requested": +// Width == 0 - no Width requested +// Width < 0 - left-justify S within and pad it to -Width chars, if necessary +// Width > 0 - right-justify S, not implemented yet +static int appendString(char **Buffer, const char *BufferEnd, int Width, + int MaxChars, const char *S) { + if (!S) + S = "<null>"; + int Res = 0; + for (; *S; S++) { + if (MaxChars >= 0 && Res >= MaxChars) + break; + Res += appendChar(Buffer, BufferEnd, *S); + } + // Only the left justified strings are supported. + while (Width < -Res) + Res += appendChar(Buffer, BufferEnd, ' '); + return Res; +} + +static int appendPointer(char **Buffer, const char *BufferEnd, u64 ptr_value) { + int Res = 0; + Res += appendString(Buffer, BufferEnd, 0, -1, "0x"); + Res += appendUnsigned(Buffer, BufferEnd, ptr_value, 16, + SCUDO_POINTER_FORMAT_LENGTH, /*PadWithZero=*/true, + /*Upper=*/false); + return Res; +} + +int formatString(char *Buffer, uptr BufferLength, const char *Format, + va_list Args) { + UNUSED static const char *PrintfFormatsHelp = + "Supported formatString formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; " + "%[-]([0-9]*)?(\\.\\*)?s; %c\n"; + RAW_CHECK(Format); + RAW_CHECK(BufferLength > 0); + const char *BufferEnd = &Buffer[BufferLength - 1]; + const char *Cur = Format; + int Res = 0; + for (; *Cur; Cur++) { + if (*Cur != '%') { + Res += appendChar(&Buffer, BufferEnd, *Cur); + continue; + } + Cur++; + const bool LeftJustified = *Cur == '-'; + if (LeftJustified) + Cur++; + bool HaveWidth = (*Cur >= '0' && *Cur <= '9'); + const bool PadWithZero = (*Cur == '0'); + u8 Width = 0; + if (HaveWidth) { + while (*Cur >= '0' && *Cur <= '9') + Width = static_cast<u8>(Width * 10 + *Cur++ - '0'); + } + const bool HavePrecision = (Cur[0] == '.' && Cur[1] == '*'); + int Precision = -1; + if (HavePrecision) { + Cur += 2; + Precision = va_arg(Args, int); + } + const bool HaveZ = (*Cur == 'z'); + Cur += HaveZ; + const bool HaveLL = !HaveZ && (Cur[0] == 'l' && Cur[1] == 'l'); + Cur += HaveLL * 2; + s64 DVal; + u64 UVal; + const bool HaveLength = HaveZ || HaveLL; + const bool HaveFlags = HaveWidth || HaveLength; + // At the moment only %s supports precision and left-justification. + CHECK(!((Precision >= 0 || LeftJustified) && *Cur != 's')); + switch (*Cur) { + case 'd': { + DVal = HaveLL ? va_arg(Args, s64) + : HaveZ ? va_arg(Args, sptr) : va_arg(Args, int); + Res += appendSignedDecimal(&Buffer, BufferEnd, DVal, Width, PadWithZero); + break; + } + case 'u': + case 'x': + case 'X': { + UVal = HaveLL ? va_arg(Args, u64) + : HaveZ ? va_arg(Args, uptr) : va_arg(Args, unsigned); + const bool Upper = (*Cur == 'X'); + Res += appendUnsigned(&Buffer, BufferEnd, UVal, (*Cur == 'u') ? 10 : 16, + Width, PadWithZero, Upper); + break; + } + case 'p': { + RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp); + Res += appendPointer(&Buffer, BufferEnd, va_arg(Args, uptr)); + break; + } + case 's': { + RAW_CHECK_MSG(!HaveLength, PrintfFormatsHelp); + // Only left-justified Width is supported. + CHECK(!HaveWidth || LeftJustified); + Res += appendString(&Buffer, BufferEnd, LeftJustified ? -Width : Width, + Precision, va_arg(Args, char *)); + break; + } + case 'c': { + RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp); + Res += + appendChar(&Buffer, BufferEnd, static_cast<char>(va_arg(Args, int))); + break; + } + case '%': { + RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp); + Res += appendChar(&Buffer, BufferEnd, '%'); + break; + } + default: { + RAW_CHECK_MSG(false, PrintfFormatsHelp); + } + } + } + RAW_CHECK(Buffer <= BufferEnd); + appendChar(&Buffer, BufferEnd + 1, '\0'); + return Res; +} + +void ScopedString::append(const char *Format, va_list Args) { + CHECK_LT(Length, String.size()); + formatString(String.data() + Length, String.size() - Length, Format, Args); + Length += strlen(String.data() + Length); + CHECK_LT(Length, String.size()); +} + +FORMAT(2, 3) +void ScopedString::append(const char *Format, ...) { + va_list Args; + va_start(Args, Format); + append(Format, Args); + va_end(Args); +} + +FORMAT(1, 2) +void Printf(const char *Format, ...) { + va_list Args; + va_start(Args, Format); + ScopedString Msg(512); + Msg.append(Format, Args); + outputRaw(Msg.data()); + va_end(Args); +} + +} // namespace scudo diff --git a/lib/scudo/standalone/string_utils.h b/lib/scudo/standalone/string_utils.h new file mode 100644 index 000000000000..aea7b3ffd7a5 --- /dev/null +++ b/lib/scudo/standalone/string_utils.h @@ -0,0 +1,42 @@ +//===-- string_utils.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_STRING_UTILS_H_ +#define SCUDO_STRING_UTILS_H_ + +#include "internal_defs.h" +#include "vector.h" + +#include <stdarg.h> + +namespace scudo { + +class ScopedString { +public: + explicit ScopedString(uptr MaxLength) : String(MaxLength), Length(0) { + String[0] = '\0'; + } + uptr length() { return Length; } + const char *data() { return String.data(); } + void clear() { + String[0] = '\0'; + Length = 0; + } + void append(const char *Format, va_list Args); + void append(const char *Format, ...); + +private: + Vector<char> String; + uptr Length; +}; + +void Printf(const char *Format, ...); + +} // namespace scudo + +#endif // SCUDO_STRING_UTILS_H_ diff --git a/lib/scudo/standalone/tsd.h b/lib/scudo/standalone/tsd.h new file mode 100644 index 000000000000..f24ff01960fb --- /dev/null +++ b/lib/scudo/standalone/tsd.h @@ -0,0 +1,66 @@ +//===-- tsd.h ---------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +#define SCUDO_TSD_H_ + +#include "atomic_helpers.h" +#include "common.h" +#include "mutex.h" + +#include <limits.h> // for PTHREAD_DESTRUCTOR_ITERATIONS + +// With some build setups, this might still not be defined. +#ifndef PTHREAD_DESTRUCTOR_ITERATIONS +#define PTHREAD_DESTRUCTOR_ITERATIONS 4 +#endif + +namespace scudo { + +template <class Allocator> struct ALIGNED(SCUDO_CACHE_LINE_SIZE) TSD { + typename Allocator::CacheT Cache; + typename Allocator::QuarantineCacheT QuarantineCache; + u8 DestructorIterations; + + void initLinkerInitialized(Allocator *Instance) { + Instance->initCache(&Cache); + DestructorIterations = PTHREAD_DESTRUCTOR_ITERATIONS; + } + void init(Allocator *Instance) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(Instance); + } + + void commitBack(Allocator *Instance) { Instance->commitBack(this); } + + INLINE bool tryLock() { + if (Mutex.tryLock()) { + atomic_store_relaxed(&Precedence, 0); + return true; + } + if (atomic_load_relaxed(&Precedence) == 0) + atomic_store_relaxed( + &Precedence, + static_cast<uptr>(getMonotonicTime() >> FIRST_32_SECOND_64(16, 0))); + return false; + } + INLINE void lock() { + atomic_store_relaxed(&Precedence, 0); + Mutex.lock(); + } + INLINE void unlock() { Mutex.unlock(); } + INLINE uptr getPrecedence() { return atomic_load_relaxed(&Precedence); } + +private: + HybridMutex Mutex; + atomic_uptr Precedence; +}; + +} // namespace scudo + +#endif // SCUDO_TSD_H_ diff --git a/lib/scudo/standalone/tsd_exclusive.h b/lib/scudo/standalone/tsd_exclusive.h new file mode 100644 index 000000000000..18cce1c56af8 --- /dev/null +++ b/lib/scudo/standalone/tsd_exclusive.h @@ -0,0 +1,118 @@ +//===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_EXCLUSIVE_H_ +#define SCUDO_TSD_EXCLUSIVE_H_ + +#include "tsd.h" + +#include <pthread.h> + +namespace scudo { + +enum class ThreadState : u8 { + NotInitialized = 0, + Initialized, + TornDown, +}; + +template <class Allocator> void teardownThread(void *Ptr); + +template <class Allocator> struct TSDRegistryExT { + void initLinkerInitialized(Allocator *Instance) { + Instance->initLinkerInitialized(); + CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0); + FallbackTSD = reinterpret_cast<TSD<Allocator> *>( + map(nullptr, sizeof(TSD<Allocator>), "scudo:tsd")); + FallbackTSD->initLinkerInitialized(Instance); + Initialized = true; + } + void init(Allocator *Instance) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(Instance); + } + + void unmapTestOnly() { + unmap(reinterpret_cast<void *>(FallbackTSD), sizeof(TSD<Allocator>)); + } + + ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) { + if (LIKELY(State != ThreadState::NotInitialized)) + return; + initThread(Instance, MinimalInit); + } + + ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) { + if (LIKELY(State == ThreadState::Initialized)) { + *UnlockRequired = false; + return &ThreadTSD; + } + DCHECK(FallbackTSD); + FallbackTSD->lock(); + *UnlockRequired = true; + return FallbackTSD; + } + +private: + void initOnceMaybe(Allocator *Instance) { + ScopedLock L(Mutex); + if (Initialized) + return; + initLinkerInitialized(Instance); // Sets Initialized. + } + + // Using minimal initialization allows for global initialization while keeping + // the thread specific structure untouched. The fallback structure will be + // used instead. + NOINLINE void initThread(Allocator *Instance, bool MinimalInit) { + initOnceMaybe(Instance); + if (MinimalInit) + return; + CHECK_EQ( + pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0); + ThreadTSD.initLinkerInitialized(Instance); + State = ThreadState::Initialized; + } + + pthread_key_t PThreadKey; + bool Initialized; + TSD<Allocator> *FallbackTSD; + HybridMutex Mutex; + static THREADLOCAL ThreadState State; + static THREADLOCAL TSD<Allocator> ThreadTSD; + + friend void teardownThread<Allocator>(void *Ptr); +}; + +template <class Allocator> +THREADLOCAL TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD; +template <class Allocator> +THREADLOCAL ThreadState TSDRegistryExT<Allocator>::State; + +template <class Allocator> void teardownThread(void *Ptr) { + typedef TSDRegistryExT<Allocator> TSDRegistryT; + Allocator *Instance = reinterpret_cast<Allocator *>(Ptr); + // The glibc POSIX thread-local-storage deallocation routine calls user + // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. + // We want to be called last since other destructors might call free and the + // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the + // quarantine and swallowing the cache. + if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) { + TSDRegistryT::ThreadTSD.DestructorIterations--; + // If pthread_setspecific fails, we will go ahead with the teardown. + if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey, + Ptr) == 0)) + return; + } + TSDRegistryT::ThreadTSD.commitBack(Instance); + TSDRegistryT::State = ThreadState::TornDown; +} + +} // namespace scudo + +#endif // SCUDO_TSD_EXCLUSIVE_H_ diff --git a/lib/scudo/standalone/tsd_shared.h b/lib/scudo/standalone/tsd_shared.h new file mode 100644 index 000000000000..0f0a83a3eed4 --- /dev/null +++ b/lib/scudo/standalone/tsd_shared.h @@ -0,0 +1,169 @@ +//===-- tsd_shared.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_SHARED_H_ +#define SCUDO_TSD_SHARED_H_ + +#include "linux.h" // for getAndroidTlsPtr() +#include "tsd.h" + +#include <pthread.h> + +namespace scudo { + +template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT { + void initLinkerInitialized(Allocator *Instance) { + Instance->initLinkerInitialized(); + CHECK_EQ(pthread_key_create(&PThreadKey, nullptr), 0); // For non-TLS + NumberOfTSDs = Min(Max(1U, getNumberOfCPUs()), MaxTSDCount); + TSDs = reinterpret_cast<TSD<Allocator> *>( + map(nullptr, sizeof(TSD<Allocator>) * NumberOfTSDs, "scudo:tsd")); + for (u32 I = 0; I < NumberOfTSDs; I++) + TSDs[I].initLinkerInitialized(Instance); + // Compute all the coprimes of NumberOfTSDs. This will be used to walk the + // array of TSDs in a random order. For details, see: + // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/ + for (u32 I = 0; I < NumberOfTSDs; I++) { + u32 A = I + 1; + u32 B = NumberOfTSDs; + // Find the GCD between I + 1 and NumberOfTSDs. If 1, they are coprimes. + while (B != 0) { + const u32 T = A; + A = B; + B = T % B; + } + if (A == 1) + CoPrimes[NumberOfCoPrimes++] = I + 1; + } + Initialized = true; + } + void init(Allocator *Instance) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(Instance); + } + + void unmapTestOnly() { + unmap(reinterpret_cast<void *>(TSDs), + sizeof(TSD<Allocator>) * NumberOfTSDs); + } + + ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, + UNUSED bool MinimalInit) { + if (LIKELY(getCurrentTSD())) + return; + initThread(Instance); + } + + ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) { + TSD<Allocator> *TSD = getCurrentTSD(); + DCHECK(TSD); + *UnlockRequired = true; + // Try to lock the currently associated context. + if (TSD->tryLock()) + return TSD; + // If that fails, go down the slow path. + return getTSDAndLockSlow(TSD); + } + +private: + ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) { +#if SCUDO_ANDROID + *getAndroidTlsPtr() = reinterpret_cast<uptr>(CurrentTSD); +#elif SCUDO_LINUX + ThreadTSD = CurrentTSD; +#else + CHECK_EQ( + pthread_setspecific(PThreadKey, reinterpret_cast<void *>(CurrentTSD)), + 0); +#endif + } + + ALWAYS_INLINE TSD<Allocator> *getCurrentTSD() { +#if SCUDO_ANDROID + return reinterpret_cast<TSD<Allocator> *>(*getAndroidTlsPtr()); +#elif SCUDO_LINUX + return ThreadTSD; +#else + return reinterpret_cast<TSD<Allocator> *>(pthread_getspecific(PThreadKey)); +#endif + } + + void initOnceMaybe(Allocator *Instance) { + ScopedLock L(Mutex); + if (Initialized) + return; + initLinkerInitialized(Instance); // Sets Initialized. + } + + NOINLINE void initThread(Allocator *Instance) { + initOnceMaybe(Instance); + // Initial context assignment is done in a plain round-robin fashion. + const u32 Index = atomic_fetch_add(&CurrentIndex, 1U, memory_order_relaxed); + setCurrentTSD(&TSDs[Index % NumberOfTSDs]); + } + + NOINLINE TSD<Allocator> *getTSDAndLockSlow(TSD<Allocator> *CurrentTSD) { + if (MaxTSDCount > 1U && NumberOfTSDs > 1U) { + // Use the Precedence of the current TSD as our random seed. Since we are + // in the slow path, it means that tryLock failed, and as a result it's + // very likely that said Precedence is non-zero. + u32 RandState = static_cast<u32>(CurrentTSD->getPrecedence()); + const u32 R = getRandomU32(&RandState); + const u32 Inc = CoPrimes[R % NumberOfCoPrimes]; + u32 Index = R % NumberOfTSDs; + uptr LowestPrecedence = UINTPTR_MAX; + TSD<Allocator> *CandidateTSD = nullptr; + // Go randomly through at most 4 contexts and find a candidate. + for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) { + if (TSDs[Index].tryLock()) { + setCurrentTSD(&TSDs[Index]); + return &TSDs[Index]; + } + const uptr Precedence = TSDs[Index].getPrecedence(); + // A 0 precedence here means another thread just locked this TSD. + if (Precedence && Precedence < LowestPrecedence) { + CandidateTSD = &TSDs[Index]; + LowestPrecedence = Precedence; + } + Index += Inc; + if (Index >= NumberOfTSDs) + Index -= NumberOfTSDs; + } + if (CandidateTSD) { + CandidateTSD->lock(); + setCurrentTSD(CandidateTSD); + return CandidateTSD; + } + } + // Last resort, stick with the current one. + CurrentTSD->lock(); + return CurrentTSD; + } + + pthread_key_t PThreadKey; + atomic_u32 CurrentIndex; + u32 NumberOfTSDs; + TSD<Allocator> *TSDs; + u32 NumberOfCoPrimes; + u32 CoPrimes[MaxTSDCount]; + bool Initialized; + HybridMutex Mutex; +#if SCUDO_LINUX && !SCUDO_ANDROID + static THREADLOCAL TSD<Allocator> *ThreadTSD; +#endif +}; + +#if SCUDO_LINUX && !SCUDO_ANDROID +template <class Allocator, u32 MaxTSDCount> +THREADLOCAL TSD<Allocator> + *TSDRegistrySharedT<Allocator, MaxTSDCount>::ThreadTSD; +#endif + +} // namespace scudo + +#endif // SCUDO_TSD_SHARED_H_ diff --git a/lib/scudo/standalone/vector.h b/lib/scudo/standalone/vector.h new file mode 100644 index 000000000000..3cb4005ed29c --- /dev/null +++ b/lib/scudo/standalone/vector.h @@ -0,0 +1,118 @@ +//===-- vector.h ------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_VECTOR_H_ +#define SCUDO_VECTOR_H_ + +#include "common.h" + +#include <string.h> + +namespace scudo { + +// A low-level vector based on map. May incur a significant memory overhead for +// small vectors. The current implementation supports only POD types. +template <typename T> class VectorNoCtor { +public: + void init(uptr InitialCapacity) { + CapacityBytes = 0; + Size = 0; + Data = nullptr; + reserve(InitialCapacity); + } + void destroy() { + if (Data) + unmap(Data, CapacityBytes); + } + T &operator[](uptr I) { + DCHECK_LT(I, Size); + return Data[I]; + } + const T &operator[](uptr I) const { + DCHECK_LT(I, Size); + return Data[I]; + } + void push_back(const T &Element) { + DCHECK_LE(Size, capacity()); + if (Size == capacity()) { + const uptr NewCapacity = roundUpToPowerOfTwo(Size + 1); + reallocate(NewCapacity); + } + memcpy(&Data[Size++], &Element, sizeof(T)); + } + T &back() { + DCHECK_GT(Size, 0); + return Data[Size - 1]; + } + void pop_back() { + DCHECK_GT(Size, 0); + Size--; + } + uptr size() const { return Size; } + const T *data() const { return Data; } + T *data() { return Data; } + uptr capacity() const { return CapacityBytes / sizeof(T); } + void reserve(uptr NewSize) { + // Never downsize internal buffer. + if (NewSize > capacity()) + reallocate(NewSize); + } + void resize(uptr NewSize) { + if (NewSize > Size) { + reserve(NewSize); + memset(&Data[Size], 0, sizeof(T) * (NewSize - Size)); + } + Size = NewSize; + } + + void clear() { Size = 0; } + bool empty() const { return size() == 0; } + + const T *begin() const { return data(); } + T *begin() { return data(); } + const T *end() const { return data() + size(); } + T *end() { return data() + size(); } + +private: + void reallocate(uptr NewCapacity) { + DCHECK_GT(NewCapacity, 0); + DCHECK_LE(Size, NewCapacity); + const uptr NewCapacityBytes = + roundUpTo(NewCapacity * sizeof(T), getPageSizeCached()); + T *NewData = (T *)map(nullptr, NewCapacityBytes, "scudo:vector"); + if (Data) { + memcpy(NewData, Data, Size * sizeof(T)); + unmap(Data, CapacityBytes); + } + Data = NewData; + CapacityBytes = NewCapacityBytes; + } + + T *Data; + uptr CapacityBytes; + uptr Size; +}; + +template <typename T> class Vector : public VectorNoCtor<T> { +public: + Vector() { VectorNoCtor<T>::init(1); } + explicit Vector(uptr Count) { + VectorNoCtor<T>::init(Count); + this->resize(Count); + } + ~Vector() { VectorNoCtor<T>::destroy(); } + // Disallow copies and moves. + Vector(const Vector &) = delete; + Vector &operator=(const Vector &) = delete; + Vector(Vector &&) = delete; + Vector &operator=(Vector &&) = delete; +}; + +} // namespace scudo + +#endif // SCUDO_VECTOR_H_ diff --git a/lib/scudo/standalone/wrappers_c.cc b/lib/scudo/standalone/wrappers_c.cc new file mode 100644 index 000000000000..5908c600be33 --- /dev/null +++ b/lib/scudo/standalone/wrappers_c.cc @@ -0,0 +1,39 @@ +//===-- wrappers_c.cc -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +// Skip this compilation unit if compiled as part of Bionic. +#if !SCUDO_ANDROID || !_BIONIC + +#include "allocator_config.h" +#include "wrappers_c.h" +#include "wrappers_c_checks.h" + +#include <stdint.h> +#include <stdio.h> + +static scudo::Allocator<scudo::Config> Allocator; +// Pointer to the static allocator so that the C++ wrappers can access it. +// Technically we could have a completely separated heap for C & C++ but in +// reality the amount of cross pollination between the two is staggering. +scudo::Allocator<scudo::Config> *AllocatorPtr = &Allocator; + +extern "C" { + +#define SCUDO_PREFIX(name) name +#define SCUDO_ALLOCATOR Allocator +#include "wrappers_c.inc" +#undef SCUDO_ALLOCATOR +#undef SCUDO_PREFIX + +INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); } + +} // extern "C" + +#endif // !SCUDO_ANDROID || !_BIONIC diff --git a/lib/scudo/standalone/wrappers_c.h b/lib/scudo/standalone/wrappers_c.h new file mode 100644 index 000000000000..33a0c53cec03 --- /dev/null +++ b/lib/scudo/standalone/wrappers_c.h @@ -0,0 +1,52 @@ +//===-- wrappers_c.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_WRAPPERS_C_H_ +#define SCUDO_WRAPPERS_C_H_ + +#include "platform.h" +#include "stats.h" + +// Bionic's struct mallinfo consists of size_t (mallinfo(3) uses int). +#if SCUDO_ANDROID +typedef size_t __scudo_mallinfo_data_t; +#else +typedef int __scudo_mallinfo_data_t; +#endif + +struct __scudo_mallinfo { + __scudo_mallinfo_data_t arena; + __scudo_mallinfo_data_t ordblks; + __scudo_mallinfo_data_t smblks; + __scudo_mallinfo_data_t hblks; + __scudo_mallinfo_data_t hblkhd; + __scudo_mallinfo_data_t usmblks; + __scudo_mallinfo_data_t fsmblks; + __scudo_mallinfo_data_t uordblks; + __scudo_mallinfo_data_t fordblks; + __scudo_mallinfo_data_t keepcost; +}; + +// Android sometimes includes malloc.h no matter what, which yields to +// conflicting return types for mallinfo() if we use our own structure. So if +// struct mallinfo is declared (#define courtesy of malloc.h), use it directly. +#if STRUCT_MALLINFO_DECLARED +#define SCUDO_MALLINFO mallinfo +#else +#define SCUDO_MALLINFO __scudo_mallinfo +#endif + +#ifndef M_DECAY_TIME +#define M_DECAY_TIME -100 +#endif + +#ifndef M_PURGE +#define M_PURGE -101 +#endif + +#endif // SCUDO_WRAPPERS_C_H_ diff --git a/lib/scudo/standalone/wrappers_c.inc b/lib/scudo/standalone/wrappers_c.inc new file mode 100644 index 000000000000..2beddc724800 --- /dev/null +++ b/lib/scudo/standalone/wrappers_c.inc @@ -0,0 +1,176 @@ +//===-- wrappers_c.inc ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PREFIX +#error "Define SCUDO_PREFIX prior to including this file!" +#endif + +// malloc-type functions have to be aligned to std::max_align_t. This is +// distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions +// do not have to abide by the same requirement. +#ifndef SCUDO_MALLOC_ALIGNMENT +#define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U) +#endif + +INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) { + scudo::uptr Product; + if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) { + if (SCUDO_ALLOCATOR.canReturnNull()) { + errno = ENOMEM; + return nullptr; + } + scudo::reportCallocOverflow(nmemb, size); + } + return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate( + Product, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT, true)); +} + +INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) { + SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc); +} + +INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) { + struct SCUDO_MALLINFO Info = {}; + scudo::StatCounters Stats; + SCUDO_ALLOCATOR.getStats(Stats); + Info.uordblks = + static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatAllocated]); + return Info; +} + +INTERFACE WEAK void *SCUDO_PREFIX(malloc)(size_t size) { + return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate( + size, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT)); +} + +#if SCUDO_ANDROID +INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(const void *ptr) { +#else +INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(void *ptr) { +#endif + return SCUDO_ALLOCATOR.getUsableSize(ptr); +} + +INTERFACE WEAK void *SCUDO_PREFIX(memalign)(size_t alignment, size_t size) { + // Android rounds up the alignment to a power of two if it isn't one. + if (SCUDO_ANDROID) { + if (UNLIKELY(!alignment)) { + alignment = 1U; + } else { + if (UNLIKELY(!scudo::isPowerOfTwo(alignment))) + alignment = scudo::roundUpToPowerOfTwo(alignment); + } + } else { + if (UNLIKELY(!scudo::isPowerOfTwo(alignment))) { + if (SCUDO_ALLOCATOR.canReturnNull()) { + errno = EINVAL; + return nullptr; + } + scudo::reportAlignmentNotPowerOfTwo(alignment); + } + } + return SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Memalign, + alignment); +} + +INTERFACE WEAK int SCUDO_PREFIX(posix_memalign)(void **memptr, size_t alignment, + size_t size) { + if (UNLIKELY(scudo::checkPosixMemalignAlignment(alignment))) { + if (!SCUDO_ALLOCATOR.canReturnNull()) + scudo::reportInvalidPosixMemalignAlignment(alignment); + return EINVAL; + } + void *Ptr = + SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Memalign, alignment); + if (UNLIKELY(!Ptr)) + return ENOMEM; + *memptr = Ptr; + return 0; +} + +INTERFACE WEAK void *SCUDO_PREFIX(pvalloc)(size_t size) { + const scudo::uptr PageSize = scudo::getPageSizeCached(); + if (UNLIKELY(scudo::checkForPvallocOverflow(size, PageSize))) { + if (SCUDO_ALLOCATOR.canReturnNull()) { + errno = ENOMEM; + return nullptr; + } + scudo::reportPvallocOverflow(size); + } + // pvalloc(0) should allocate one page. + return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate( + size ? scudo::roundUpTo(size, PageSize) : PageSize, + scudo::Chunk::Origin::Memalign, PageSize)); +} + +INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) { + if (!ptr) + return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate( + size, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT)); + if (size == 0) { + SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc); + return nullptr; + } + return scudo::setErrnoOnNull( + SCUDO_ALLOCATOR.reallocate(ptr, size, SCUDO_MALLOC_ALIGNMENT)); +} + +INTERFACE WEAK void *SCUDO_PREFIX(valloc)(size_t size) { + return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate( + size, scudo::Chunk::Origin::Memalign, scudo::getPageSizeCached())); +} + +// Bionic wants a function named PREFIX_iterate and not PREFIX_malloc_iterate +// which is somewhat inconsistent with the rest, workaround that. +#if SCUDO_ANDROID && _BIONIC +#define SCUDO_ITERATE iterate +#else +#define SCUDO_ITERATE malloc_iterate +#endif + +INTERFACE WEAK int SCUDO_PREFIX(SCUDO_ITERATE)( + uintptr_t base, size_t size, + void (*callback)(uintptr_t base, size_t size, void *arg), void *arg) { + SCUDO_ALLOCATOR.iterateOverChunks(base, size, callback, arg); + return 0; +} + +INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() { + SCUDO_ALLOCATOR.disable(); +} + +INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { SCUDO_ALLOCATOR.enable(); } + +INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) { + if (param == M_DECAY_TIME) { + // TODO(kostyak): set release_to_os_interval_ms accordingly. + return 1; + } else if (param == M_PURGE) { + SCUDO_ALLOCATOR.releaseToOS(); + return 1; + } + return 0; +} + +INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment, + size_t size) { + if (UNLIKELY(scudo::checkAlignedAllocAlignmentAndSize(alignment, size))) { + if (SCUDO_ALLOCATOR.canReturnNull()) { + errno = EINVAL; + return nullptr; + } + scudo::reportInvalidAlignedAllocAlignment(alignment, size); + } + return scudo::setErrnoOnNull( + SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Malloc, alignment)); +} + +INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(int, FILE *) { + errno = ENOTSUP; + return -1; +} diff --git a/lib/scudo/standalone/wrappers_c_bionic.cc b/lib/scudo/standalone/wrappers_c_bionic.cc new file mode 100644 index 000000000000..f6e863deb973 --- /dev/null +++ b/lib/scudo/standalone/wrappers_c_bionic.cc @@ -0,0 +1,49 @@ +//===-- wrappers_c_bionic.cc ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +// This is only used when compiled as part of Bionic. +#if SCUDO_ANDROID && _BIONIC + +#include "allocator_config.h" +#include "wrappers_c.h" +#include "wrappers_c_checks.h" + +#include <stdint.h> +#include <stdio.h> + +static scudo::Allocator<scudo::AndroidConfig> Allocator; +static scudo::Allocator<scudo::AndroidSvelteConfig> SvelteAllocator; + +extern "C" { + +// Regular MallocDispatch definitions. +#define SCUDO_PREFIX(name) CONCATENATE(scudo_, name) +#define SCUDO_ALLOCATOR Allocator +#include "wrappers_c.inc" +#undef SCUDO_ALLOCATOR +#undef SCUDO_PREFIX + +// Svelte MallocDispatch definitions. +#define SCUDO_PREFIX(name) CONCATENATE(scudo_svelte_, name) +#define SCUDO_ALLOCATOR SvelteAllocator +#include "wrappers_c.inc" +#undef SCUDO_ALLOCATOR +#undef SCUDO_PREFIX + +// The following is the only function that will end up initializing both +// allocators, which will result in a slight increase in memory footprint. +INTERFACE void __scudo_print_stats(void) { + Allocator.printStats(); + SvelteAllocator.printStats(); +} + +} // extern "C" + +#endif // SCUDO_ANDROID && _BIONIC diff --git a/lib/scudo/standalone/wrappers_c_checks.h b/lib/scudo/standalone/wrappers_c_checks.h new file mode 100644 index 000000000000..d4370d506e5e --- /dev/null +++ b/lib/scudo/standalone/wrappers_c_checks.h @@ -0,0 +1,67 @@ +//===-- wrappers_c_checks.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CHECKS_H_ +#define SCUDO_CHECKS_H_ + +#include "common.h" + +#include <errno.h> + +#ifndef __has_builtin +#define __has_builtin(X) 0 +#endif + +namespace scudo { + +// A common errno setting logic shared by almost all Scudo C wrappers. +INLINE void *setErrnoOnNull(void *Ptr) { + if (UNLIKELY(!Ptr)) + errno = ENOMEM; + return Ptr; +} + +// Checks return true on failure. + +// Checks aligned_alloc() parameters, verifies that the alignment is a power of +// two and that the size is a multiple of alignment. +INLINE bool checkAlignedAllocAlignmentAndSize(uptr Alignment, uptr Size) { + return Alignment == 0 || !isPowerOfTwo(Alignment) || + !isAligned(Size, Alignment); +} + +// Checks posix_memalign() parameters, verifies that alignment is a power of two +// and a multiple of sizeof(void *). +INLINE bool checkPosixMemalignAlignment(uptr Alignment) { + return Alignment == 0 || !isPowerOfTwo(Alignment) || + !isAligned(Alignment, sizeof(void *)); +} + +// Returns true if calloc(Size, N) overflows on Size*N calculation. Use a +// builtin supported by recent clang & GCC if it exists, otherwise fallback to a +// costly division. +INLINE bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) { +#if __has_builtin(__builtin_umull_overflow) + return __builtin_umull_overflow(Size, N, Product); +#else + *Product = Size * N; + if (!Size) + return false; + return (*Product / Size) != N; +#endif +} + +// Returns true if the size passed to pvalloc overflows when rounded to the next +// multiple of PageSize. +INLINE bool checkForPvallocOverflow(uptr Size, uptr PageSize) { + return roundUpTo(Size, PageSize) < Size; +} + +} // namespace scudo + +#endif // SCUDO_CHECKS_H_ diff --git a/lib/scudo/standalone/wrappers_cpp.cc b/lib/scudo/standalone/wrappers_cpp.cc new file mode 100644 index 000000000000..3ae1cdc05a06 --- /dev/null +++ b/lib/scudo/standalone/wrappers_cpp.cc @@ -0,0 +1,107 @@ +//===-- wrappers_cpp.cc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +// Skip this compilation unit if compiled as part of Bionic. +#if !SCUDO_ANDROID || !_BIONIC + +#include "allocator_config.h" + +#include <stdint.h> + +extern scudo::Allocator<scudo::Config> *AllocatorPtr; + +namespace std { +struct nothrow_t {}; +enum class align_val_t : size_t {}; +} // namespace std + +INTERFACE WEAK void *operator new(size_t size) { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New); +} +INTERFACE WEAK void *operator new[](size_t size) { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray); +} +INTERFACE WEAK void *operator new(size_t size, + std::nothrow_t const &) NOEXCEPT { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New); +} +INTERFACE WEAK void *operator new[](size_t size, + std::nothrow_t const &) NOEXCEPT { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray); +} +INTERFACE WEAK void *operator new(size_t size, std::align_val_t align) { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void *operator new[](size_t size, std::align_val_t align) { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void *operator new(size_t size, std::align_val_t align, + std::nothrow_t const &) NOEXCEPT { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void *operator new[](size_t size, std::align_val_t align, + std::nothrow_t const &) NOEXCEPT { + return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray, + static_cast<scudo::uptr>(align)); +} + +INTERFACE WEAK void operator delete(void *ptr)NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New); +} +INTERFACE WEAK void operator delete[](void *ptr) NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray); +} +INTERFACE WEAK void operator delete(void *ptr, std::nothrow_t const &)NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New); +} +INTERFACE WEAK void operator delete[](void *ptr, + std::nothrow_t const &) NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray); +} +INTERFACE WEAK void operator delete(void *ptr, size_t size)NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, size); +} +INTERFACE WEAK void operator delete[](void *ptr, size_t size) NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, size); +} +INTERFACE WEAK void operator delete(void *ptr, std::align_val_t align)NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, 0, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void operator delete[](void *ptr, + std::align_val_t align) NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, 0, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void operator delete(void *ptr, std::align_val_t align, + std::nothrow_t const &)NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, 0, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void operator delete[](void *ptr, std::align_val_t align, + std::nothrow_t const &) NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, 0, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void operator delete(void *ptr, size_t size, + std::align_val_t align)NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, size, + static_cast<scudo::uptr>(align)); +} +INTERFACE WEAK void operator delete[](void *ptr, size_t size, + std::align_val_t align) NOEXCEPT { + AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, size, + static_cast<scudo::uptr>(align)); +} + +#endif // !SCUDO_ANDROID || !_BIONIC diff --git a/lib/stats/stats.cc b/lib/stats/stats.cc index c7a9a38a612a..8d4a115d1aef 100644 --- a/lib/stats/stats.cc +++ b/lib/stats/stats.cc @@ -1,9 +1,8 @@ //===-- stats.cc ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/stats/stats.h b/lib/stats/stats.h index 61947067773a..4b7514d61082 100644 --- a/lib/stats/stats.h +++ b/lib/stats/stats.h @@ -1,9 +1,8 @@ //===-- stats.h -------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/stats/stats_client.cc b/lib/stats/stats_client.cc index 5caf09728f73..0790f57372f4 100644 --- a/lib/stats/stats_client.cc +++ b/lib/stats/stats_client.cc @@ -1,9 +1,8 @@ //===-- stats_client.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/benchmarks/func_entry_exit.cc b/lib/tsan/benchmarks/func_entry_exit.cc new file mode 100644 index 000000000000..5e0ba1d6981b --- /dev/null +++ b/lib/tsan/benchmarks/func_entry_exit.cc @@ -0,0 +1,20 @@ +// Synthetic benchmark for __tsan_func_entry/exit (spends ~75% there). + +void foo(bool x); + +int main() { + volatile int kRepeat1 = 1 << 30; + const int kRepeat = kRepeat1; + for (int i = 0; i < kRepeat; i++) + foo(false); +} + +__attribute__((noinline)) void bar(volatile bool x) { + if (x) + foo(x); +} + +__attribute__((noinline)) void foo(bool x) { + if (__builtin_expect(x, false)) + bar(x); +} diff --git a/lib/tsan/benchmarks/mop.cc b/lib/tsan/benchmarks/mop.cc new file mode 100644 index 000000000000..e87fab856969 --- /dev/null +++ b/lib/tsan/benchmarks/mop.cc @@ -0,0 +1,80 @@ +// Synthetic benchmark for __tsan_read/write{1,2,4,8}. +// As compared to mini_bench_local/shared.cc this benchmark passes through +// deduplication logic (ContainsSameAccess). +// First argument is access size (1, 2, 4, 8). Second optional arg switches +// from writes to reads. + +#include <pthread.h> +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <linux/futex.h> +#include <sys/syscall.h> +#include <sys/time.h> + +template<typename T, bool write> +void* thread(void *arg) { + const int kSize = 2 << 10; + static volatile long data[kSize]; + static volatile long turn; + const int kRepeat = 1 << 17; + const int id = !!arg; + for (int i = 0; i < kRepeat; i++) { + for (;;) { + int t = __atomic_load_n(&turn, __ATOMIC_ACQUIRE); + if (t == id) + break; + syscall(SYS_futex, &turn, FUTEX_WAIT, t, 0, 0, 0); + } + for (int j = 0; j < kSize; j++) { + if (write) { + ((volatile T*)&data[j])[0] = 1; + ((volatile T*)&data[j])[sizeof(T) == 8 ? 0 : 1] = 1; + } else { + T v0 = ((volatile T*)&data[j])[0]; + T v1 = ((volatile T*)&data[j])[sizeof(T) == 8 ? 0 : 1]; + (void)v0; + (void)v1; + } + } + __atomic_store_n(&turn, 1 - id, __ATOMIC_RELEASE); + syscall(SYS_futex, &turn, FUTEX_WAKE, 0, 0, 0, 0); + } + return 0; +} + +template<typename T, bool write> +void test() { + pthread_t th; + pthread_create(&th, 0, thread<T, write>, (void*)1); + thread<T, write>(0); + pthread_join(th, 0); +} + +template<bool write> +void testw(int size) { + switch (size) { + case 1: return test<char, write>(); + case 2: return test<short, write>(); + case 4: return test<int, write>(); + case 8: return test<long long, write>(); + } +} + +int main(int argc, char** argv) { + int size = 8; + bool write = true; + if (argc > 1) { + size = atoi(argv[1]); + if (size != 1 && size != 2 && size != 4 && size != 8) + size = 8; + } + if (argc > 2) + write = false; + printf("%s%d\n", write ? "write" : "read", size); + if (write) + testw<true>(size); + else + testw<false>(size); + return 0; +} diff --git a/lib/tsan/dd/dd_interceptors.cc b/lib/tsan/dd/dd_interceptors.cc index a39218f0454b..35a72eb36a30 100644 --- a/lib/tsan/dd/dd_interceptors.cc +++ b/lib/tsan/dd/dd_interceptors.cc @@ -1,9 +1,8 @@ //===-- dd_interceptors.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/tsan/dd/dd_rtl.cc b/lib/tsan/dd/dd_rtl.cc index 99b8ee597cd1..08652ba01cdd 100644 --- a/lib/tsan/dd/dd_rtl.cc +++ b/lib/tsan/dd/dd_rtl.cc @@ -1,9 +1,8 @@ //===-- dd_rtl.cc ---------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -84,7 +83,7 @@ static void InitializeFlags() { FlagParser parser; RegisterFlag(&parser, "second_deadlock_stack", "", &f->second_deadlock_stack); RegisterCommonFlags(&parser); - parser.ParseString(GetEnv("DSAN_OPTIONS")); + parser.ParseStringFromEnv("DSAN_OPTIONS"); SetVerbosity(common_flags()->verbosity); } diff --git a/lib/tsan/dd/dd_rtl.h b/lib/tsan/dd/dd_rtl.h index 9abf17da48c8..ffe0684306dc 100644 --- a/lib/tsan/dd/dd_rtl.h +++ b/lib/tsan/dd/dd_rtl.h @@ -1,9 +1,8 @@ //===-- dd_rtl.h ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef DD_RTL_H diff --git a/lib/tsan/go/test.c b/lib/tsan/go/test.c index c484774caeb9..61be48442c80 100644 --- a/lib/tsan/go/test.c +++ b/lib/tsan/go/test.c @@ -1,9 +1,8 @@ //===-- test.c ------------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/go/tsan_go.cc b/lib/tsan/go/tsan_go.cc index 5f2507b7df1d..dfd1e1da158f 100644 --- a/lib/tsan/go/tsan_go.cc +++ b/lib/tsan/go/tsan_go.cc @@ -1,9 +1,8 @@ //===-- tsan_go.cc --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -214,7 +213,7 @@ void __tsan_go_start(ThreadState *parent, ThreadState **pthr, void *pc) { ThreadState *thr = AllocGoroutine(); *pthr = thr; int goid = ThreadCreate(parent, (uptr)pc, 0, true); - ThreadStart(thr, goid, 0, /*workerthread*/ false); + ThreadStart(thr, goid, 0, ThreadType::Regular); } void __tsan_go_end(ThreadState *thr) { diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc index ef984a45cd9d..685ca5518009 100644 --- a/lib/tsan/rtl/tsan_clock.cc +++ b/lib/tsan/rtl/tsan_clock.cc @@ -1,9 +1,8 @@ //===-- tsan_clock.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h index a891d7bbd889..6a1d15a2a16d 100644 --- a/lib/tsan/rtl/tsan_clock.h +++ b/lib/tsan/rtl/tsan_clock.h @@ -1,9 +1,8 @@ //===-- tsan_clock.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_debugging.cc b/lib/tsan/rtl/tsan_debugging.cc index 067aeef97c0e..8579db12b666 100644 --- a/lib/tsan/rtl/tsan_debugging.cc +++ b/lib/tsan/rtl/tsan_debugging.cc @@ -1,9 +1,8 @@ //===-- tsan_debugging.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h index 3c775debfb09..293d7deccc31 100644 --- a/lib/tsan/rtl/tsan_defs.h +++ b/lib/tsan/rtl/tsan_defs.h @@ -1,9 +1,8 @@ //===-- tsan_defs.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h index 16dbdf391085..64fc50e95c25 100644 --- a/lib/tsan/rtl/tsan_dense_alloc.h +++ b/lib/tsan/rtl/tsan_dense_alloc.h @@ -1,9 +1,8 @@ //===-- tsan_dense_alloc.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_dispatch_defs.h b/lib/tsan/rtl/tsan_dispatch_defs.h new file mode 100644 index 000000000000..6f1d1f75f600 --- /dev/null +++ b/lib/tsan/rtl/tsan_dispatch_defs.h @@ -0,0 +1,66 @@ +//===-- tsan_dispatch_defs.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_DISPATCH_DEFS_H +#define TSAN_DISPATCH_DEFS_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +typedef struct dispatch_object_s {} *dispatch_object_t; + +#define DISPATCH_DECL(name) \ + typedef struct name##_s : public dispatch_object_s {} *name##_t + +DISPATCH_DECL(dispatch_queue); +DISPATCH_DECL(dispatch_source); +DISPATCH_DECL(dispatch_group); +DISPATCH_DECL(dispatch_data); +DISPATCH_DECL(dispatch_semaphore); +DISPATCH_DECL(dispatch_io); + +typedef void (*dispatch_function_t)(void *arg); +typedef void (^dispatch_block_t)(void); +typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data, + int error); + +typedef long dispatch_once_t; // NOLINT +typedef __sanitizer::u64 dispatch_time_t; +typedef int dispatch_fd_t; // NOLINT +typedef unsigned long dispatch_io_type_t; // NOLINT +typedef unsigned long dispatch_io_close_flags_t; // NOLINT + +extern "C" { +void *dispatch_get_context(dispatch_object_t object); +void dispatch_retain(dispatch_object_t object); +void dispatch_release(dispatch_object_t object); + +extern const dispatch_block_t _dispatch_data_destructor_free; +extern const dispatch_block_t _dispatch_data_destructor_munmap; +} // extern "C" + +#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr +#define DISPATCH_DATA_DESTRUCTOR_FREE _dispatch_data_destructor_free +#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap + +#if __has_attribute(noescape) + #define DISPATCH_NOESCAPE __attribute__((__noescape__)) +#else + #define DISPATCH_NOESCAPE +#endif + +// Data types used in dispatch APIs +typedef unsigned long size_t; // NOLINT +typedef unsigned long uintptr_t; // NOLINT +typedef __sanitizer::s64 off_t; +typedef __sanitizer::u16 mode_t; +typedef long long_t; // NOLINT + +#endif // TSAN_DISPATCH_DEFS_H diff --git a/lib/tsan/rtl/tsan_external.cc b/lib/tsan/rtl/tsan_external.cc index 6c0e9477e41e..ba8bb71be43b 100644 --- a/lib/tsan/rtl/tsan_external.cc +++ b/lib/tsan/rtl/tsan_external.cc @@ -1,9 +1,8 @@ //===-- tsan_external.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.cc b/lib/tsan/rtl/tsan_fd.cc index f13a7432ec95..5b562ae68d5a 100644 --- a/lib/tsan/rtl/tsan_fd.cc +++ b/lib/tsan/rtl/tsan_fd.cc @@ -1,9 +1,8 @@ //===-- tsan_fd.cc --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_fd.h b/lib/tsan/rtl/tsan_fd.h index 64dc84f6edd6..ce4f2f73bac6 100644 --- a/lib/tsan/rtl/tsan_fd.h +++ b/lib/tsan/rtl/tsan_fd.h @@ -1,9 +1,8 @@ //===-- tsan_fd.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_flags.cc b/lib/tsan/rtl/tsan_flags.cc index 877fc8b819fd..a5abb183a753 100644 --- a/lib/tsan/rtl/tsan_flags.cc +++ b/lib/tsan/rtl/tsan_flags.cc @@ -1,9 +1,8 @@ //===-- tsan_flags.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -50,7 +49,7 @@ void RegisterTsanFlags(FlagParser *parser, Flags *f) { &f->second_deadlock_stack); } -void InitializeFlags(Flags *f, const char *env) { +void InitializeFlags(Flags *f, const char *env, const char *env_option_name) { SetCommonFlagsDefaults(); { // Override some common flags defaults. @@ -92,9 +91,9 @@ void InitializeFlags(Flags *f, const char *env) { ubsan_parser.ParseString(ubsan_default_options); #endif // Override from command line. - parser.ParseString(env); + parser.ParseString(env, env_option_name); #if TSAN_CONTAINS_UBSAN - ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); #endif // Sanity check. diff --git a/lib/tsan/rtl/tsan_flags.h b/lib/tsan/rtl/tsan_flags.h index 66740def52fa..da27d5b992bc 100644 --- a/lib/tsan/rtl/tsan_flags.h +++ b/lib/tsan/rtl/tsan_flags.h @@ -1,9 +1,8 @@ //===-- tsan_flags.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,8 @@ struct Flags : DDFlags { void ParseFromString(const char *str); }; -void InitializeFlags(Flags *flags, const char *env); +void InitializeFlags(Flags *flags, const char *env, + const char *env_option_name = nullptr); } // namespace __tsan #endif // TSAN_FLAGS_H diff --git a/lib/tsan/rtl/tsan_flags.inc b/lib/tsan/rtl/tsan_flags.inc index d3b678fdd151..bfb74b696e67 100644 --- a/lib/tsan/rtl/tsan_flags.inc +++ b/lib/tsan/rtl/tsan_flags.inc @@ -1,9 +1,8 @@ //===-- tsan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.cc b/lib/tsan/rtl/tsan_ignoreset.cc index cdb90d229980..b2f657939cce 100644 --- a/lib/tsan/rtl/tsan_ignoreset.cc +++ b/lib/tsan/rtl/tsan_ignoreset.cc @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_ignoreset.h b/lib/tsan/rtl/tsan_ignoreset.h index e747d819c758..3e318bd674d9 100644 --- a/lib/tsan/rtl/tsan_ignoreset.h +++ b/lib/tsan/rtl/tsan_ignoreset.h @@ -1,9 +1,8 @@ //===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index 9e49dfe5d582..9a184c797985 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -154,7 +153,7 @@ const int SIG_SETMASK = 2; #endif #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ - (!cur_thread()->is_inited) + (cur_thread_init(), !cur_thread()->is_inited) namespace __tsan { struct SignalDesc { @@ -399,7 +398,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. @@ -409,7 +408,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); @@ -455,7 +454,7 @@ static void on_exit_wrapper(int status, void *arg) { } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); @@ -487,7 +486,7 @@ static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { } } -static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { +static void SetJmp(ThreadState *thr, uptr sp) { if (!thr->is_inited) // called from libc guts during bootstrap return; // Cleanup old bufs. @@ -495,7 +494,6 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { // Remember the buf. JmpBuf *buf = thr->jmp_bufs.PushBack(); buf->sp = sp; - buf->mangled_sp = mangled_sp; buf->shadow_stack_pos = thr->shadow_stack_pos; ThreadSignalContext *sctx = SigCtx(thr); buf->int_signal_send = sctx ? sctx->int_signal_send : 0; @@ -507,32 +505,11 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { } static void LongJmp(ThreadState *thr, uptr *env) { -#ifdef __powerpc__ - uptr mangled_sp = env[0]; -#elif SANITIZER_FREEBSD - uptr mangled_sp = env[2]; -#elif SANITIZER_NETBSD - uptr mangled_sp = env[6]; -#elif SANITIZER_MAC -# ifdef __aarch64__ - uptr mangled_sp = - (GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? env[12] : env[13]; -# else - uptr mangled_sp = env[2]; -# endif -#elif SANITIZER_LINUX -# ifdef __aarch64__ - uptr mangled_sp = env[13]; -# elif defined(__mips64) - uptr mangled_sp = env[1]; -# else - uptr mangled_sp = env[6]; -# endif -#endif - // Find the saved buf by mangled_sp. + uptr sp = ExtractLongJmpSp(env); + // Find the saved buf with matching sp. for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; - if (buf->mangled_sp == mangled_sp) { + if (buf->sp == sp) { CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); // Unwind the stack. while (thr->shadow_stack_pos > buf->shadow_stack_pos) @@ -554,8 +531,9 @@ static void LongJmp(ThreadState *thr, uptr *env) { } // FIXME: put everything below into a common extern "C" block? -extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { - SetJmp(cur_thread(), sp, mangled_sp); +extern "C" void __tsan_setjmp(uptr sp) { + cur_thread_init(); + SetJmp(cur_thread(), sp); } #if SANITIZER_MAC @@ -665,7 +643,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(size); void *p = 0; { @@ -682,7 +660,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalCalloc(size, n); void *p = 0; { @@ -694,7 +672,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalRealloc(p, size); if (p) invoke_free_hook(p); @@ -706,10 +684,23 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { return p; } +TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { + if (in_symbolizer()) + return InternalReallocArray(p, size, n); + if (p) + invoke_free_hook(p); + { + SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); + p = user_reallocarray(thr, pc, p, size, n); + } + invoke_malloc_hook(p, size); + return p; +} + TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); @@ -719,7 +710,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); @@ -808,14 +799,14 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, align); SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); return user_aligned_alloc(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return InternalAlloc(sz, nullptr, GetPageSizeCached()); SCOPED_INTERCEPTOR_RAW(valloc, sz); return user_valloc(thr, pc, sz); @@ -824,7 +815,7 @@ TSAN_INTERCEPTOR(void*, valloc, uptr sz) { #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { uptr PageSize = GetPageSizeCached(); sz = sz ? RoundUpTo(sz, PageSize) : PageSize; return InternalAlloc(sz, nullptr, PageSize); @@ -839,7 +830,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { - if (UNLIKELY(cur_thread()->in_symbolizer)) { + if (in_symbolizer()) { void *p = InternalAlloc(sz, nullptr, align); if (!p) return errno_ENOMEM; @@ -943,6 +934,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { void *param = p->param; int tid = 0; { + cur_thread_init(); ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; @@ -959,7 +951,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { internal_sched_yield(); Processor *proc = ProcCreate(); ProcWire(proc, thr); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); atomic_store(&p->tid, 0, memory_order_release); } void *res = callback(param); @@ -1051,6 +1043,16 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { return res; } +TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { + { + SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); +#if !SANITIZER_MAC && !SANITIZER_ANDROID + CHECK_EQ(thr, &cur_thread_placeholder); +#endif + } + REAL(pthread_exit)(retval); +} + #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); @@ -1975,6 +1977,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, __sanitizer_siginfo *info, void *ctx) { + cur_thread_init(); ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { @@ -2091,7 +2094,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, } TSAN_INTERCEPTOR(int, fork, int fake) { - if (UNLIKELY(cur_thread()->in_symbolizer)) + if (in_symbolizer()) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); @@ -2204,23 +2207,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO -// There interceptors do not seem to be strictly necessary for tsan. -// But we see cases where the interceptors consume 70% of execution time. -// Memory blocks passed to fgetgrent_r are "written to" by tsan several times. -// First, there is some recursion (getgrnam_r calls fgetgrent_r), and each -// function "writes to" the buffer. Then, the same memory is "written to" -// twice, first as buf and then as pwbufp (both of them refer to the same -// addresses). -#undef SANITIZER_INTERCEPT_GETPWENT -#undef SANITIZER_INTERCEPT_GETPWENT_R -#undef SANITIZER_INTERCEPT_FGETPWENT -#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS -#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS // We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR +#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ @@ -2620,6 +2612,9 @@ static void unreachable() { } #endif +// Define default implementation since interception of libdispatch is optional. +SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} + void InitializeInterceptors() { #if !SANITIZER_MAC // We need to setup it early, because functions like dlsym() can call it. @@ -2637,18 +2632,18 @@ void InitializeInterceptors() { InitializeCommonInterceptors(); InitializeSignalInterceptors(); + InitializeLibdispatchInterceptors(); #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. - using __interception::GetRealFunctionAddress; - GetRealFunctionAddress(TSAN_STRING_SETJMP, - (uptr*)&REAL(setjmp_symname), 0, 0); - GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); - GetRealFunctionAddress(TSAN_STRING_SIGSETJMP, - (uptr*)&REAL(sigsetjmp_symname), 0, 0); + using __interception::InterceptFunction; + InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); + InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); + InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, + 0); #if !SANITIZER_NETBSD - GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); + InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif #endif @@ -2662,6 +2657,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(__libc_memalign); TSAN_INTERCEPT(calloc); TSAN_INTERCEPT(realloc); + TSAN_INTERCEPT(reallocarray); TSAN_INTERCEPT(free); TSAN_INTERCEPT(cfree); TSAN_INTERCEPT(munmap); @@ -2677,6 +2673,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + TSAN_INTERCEPT(pthread_exit); #if SANITIZER_LINUX TSAN_INTERCEPT(pthread_tryjoin_np); TSAN_INTERCEPT(pthread_timedjoin_np); diff --git a/lib/tsan/rtl/tsan_interceptors.h b/lib/tsan/rtl/tsan_interceptors.h index 763b46b88c8f..88d1edd775d3 100644 --- a/lib/tsan/rtl/tsan_interceptors.h +++ b/lib/tsan/rtl/tsan_interceptors.h @@ -21,9 +21,17 @@ class ScopedInterceptor { LibIgnore *libignore(); +#if !SANITIZER_GO +INLINE bool in_symbolizer() { + cur_thread_init(); + return UNLIKELY(cur_thread()->in_symbolizer); +} +#endif + } // namespace __tsan #define SCOPED_INTERCEPTOR_RAW(func, ...) \ + cur_thread_init(); \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc index 579c4d0c0b81..99c6df9dfde7 100644 --- a/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_interceptors_mac.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,8 +20,10 @@ #include "tsan_interface_ann.h" #include "sanitizer_common/sanitizer_addrhashmap.h" +#include <errno.h> #include <libkern/OSAtomic.h> #include <objc/objc-sync.h> +#include <sys/ucontext.h> #if defined(__has_include) && __has_include(<xpc/xpc.h>) #include <xpc/xpc.h> @@ -30,6 +31,11 @@ typedef long long_t; // NOLINT +extern "C" { +int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); +int setcontext(const ucontext_t *ucp); +} + namespace __tsan { // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, @@ -354,6 +360,31 @@ TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { return result; } +TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { + { + SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); + } + // Bacause of swapcontext() semantics we have no option but to copy its + // impementation here + if (!oucp || !ucp) { + errno = EINVAL; + return -1; + } + ThreadState *thr = cur_thread(); + const int UCF_SWAPPED = 0x80000000; + oucp->uc_onstack &= ~UCF_SWAPPED; + thr->ignore_interceptors++; + int ret = getcontext(oucp); + if (!(oucp->uc_onstack & UCF_SWAPPED)) { + thr->ignore_interceptors--; + if (!ret) { + oucp->uc_onstack |= UCF_SWAPPED; + ret = setcontext(ucp); + } + } + return ret; +} + // On macOS, libc++ is always linked dynamically, so intercepting works the // usual way. #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR diff --git a/lib/tsan/rtl/tsan_interface.cc b/lib/tsan/rtl/tsan_interface.cc index ad9b1fe9a585..508aadb08f62 100644 --- a/lib/tsan/rtl/tsan_interface.cc +++ b/lib/tsan/rtl/tsan_interface.cc @@ -1,9 +1,8 @@ //===-- tsan_interface.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef u32 uint32_t; typedef u64 uint64_t; void __tsan_init() { + cur_thread_init(); Initialize(cur_thread()); } @@ -124,6 +124,31 @@ void __sanitizer_unaligned_store64(uu64 *addr, u64 v) { __tsan_unaligned_write8(addr); *addr = v; } + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(cur_thread(), CALLERPC, flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber)); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber), flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast<ThreadState *>(fiber), name); +} } // extern "C" void __tsan_acquire(void *addr) { diff --git a/lib/tsan/rtl/tsan_interface.h b/lib/tsan/rtl/tsan_interface.h index 203f6b106a96..fac57809aa24 100644 --- a/lib/tsan/rtl/tsan_interface.h +++ b/lib/tsan/rtl/tsan_interface.h @@ -1,9 +1,8 @@ //===-- tsan_interface.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -200,7 +199,7 @@ __extension__ typedef __int128 a128; #endif // Part of ABI, do not change. -// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup +// https://github.com/llvm/llvm-project/blob/master/libcxx/include/atomic typedef enum { mo_relaxed, mo_consume, diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc index a7922a42e42c..e141ddbb751a 100644 --- a/lib/tsan/rtl/tsan_interface_ann.cc +++ b/lib/tsan/rtl/tsan_interface_ann.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_ann.h b/lib/tsan/rtl/tsan_interface_ann.h index 963bcc55afaa..458d61f53356 100644 --- a/lib/tsan/rtl/tsan_interface_ann.h +++ b/lib/tsan/rtl/tsan_interface_ann.h @@ -1,9 +1,8 @@ //===-- tsan_interface_ann.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc index d334394f5303..a6b7b0f656d3 100644 --- a/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/lib/tsan/rtl/tsan_interface_atomic.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_atomic.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -475,7 +474,7 @@ static morder convert_morder(morder mo) { #define SCOPED_ATOMIC(func, ...) \ ThreadState *const thr = cur_thread(); \ - if (thr->ignore_sync || thr->ignore_interceptors) { \ + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \ ProcessPendingSignals(thr); \ return NoTsanAtomic##func(__VA_ARGS__); \ } \ diff --git a/lib/tsan/rtl/tsan_interface_inl.h b/lib/tsan/rtl/tsan_interface_inl.h index fff83ee17806..bf4a1658625c 100644 --- a/lib/tsan/rtl/tsan_interface_inl.h +++ b/lib/tsan/rtl/tsan_interface_inl.h @@ -1,9 +1,8 @@ //===-- tsan_interface_inl.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.cc b/lib/tsan/rtl/tsan_interface_java.cc index 75e960e629f9..9f227f09589e 100644 --- a/lib/tsan/rtl/tsan_interface_java.cc +++ b/lib/tsan/rtl/tsan_interface_java.cc @@ -1,9 +1,8 @@ //===-- tsan_interface_java.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_interface_java.h b/lib/tsan/rtl/tsan_interface_java.h index 0bd49ac3c999..5ad16e959093 100644 --- a/lib/tsan/rtl/tsan_interface_java.h +++ b/lib/tsan/rtl/tsan_interface_java.h @@ -1,9 +1,8 @@ //===-- tsan_interface_java.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_libdispatch_mac.cc b/lib/tsan/rtl/tsan_libdispatch.cc index df22888b3f1c..48ac6a2824a8 100644 --- a/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/lib/tsan/rtl/tsan_libdispatch.cc @@ -1,38 +1,26 @@ -//===-- tsan_libdispatch_mac.cc -------------------------------------------===// +//===-- tsan_libdispatch.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // -// Mac-specific libdispatch (GCD) support. +// Support for intercepting libdispatch (GCD). //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC - #include "sanitizer_common/sanitizer_common.h" #include "interception/interception.h" #include "tsan_interceptors.h" -#include "tsan_platform.h" #include "tsan_rtl.h" -#include <Block.h> -#include <dispatch/dispatch.h> -#include <pthread.h> - -// DISPATCH_NOESCAPE is not defined prior to XCode 8. -#ifndef DISPATCH_NOESCAPE -#define DISPATCH_NOESCAPE -#endif - -typedef long long_t; // NOLINT +#include "BlocksRuntime/Block.h" +#include "tsan_dispatch_defs.h" namespace __tsan { + typedef u16 uint16_t; typedef struct { dispatch_queue_t queue; @@ -42,7 +30,7 @@ typedef struct { bool submitted_synchronously; bool is_barrier_block; uptr non_queue_sync_object; -} tsan_block_context_t; +} block_context_t; // The offsets of different fields of the dispatch_queue_t structure, exported // by libdispatch.dylib. @@ -86,13 +74,11 @@ static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { return tq; } -static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, - dispatch_queue_t queue, - void *orig_context, - dispatch_function_t orig_work) { - tsan_block_context_t *new_context = - (tsan_block_context_t *)user_alloc_internal(thr, pc, - sizeof(tsan_block_context_t)); +static block_context_t *AllocContext(ThreadState *thr, uptr pc, + dispatch_queue_t queue, void *orig_context, + dispatch_function_t orig_work) { + block_context_t *new_context = + (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); new_context->queue = queue; new_context->orig_context = orig_context; new_context->orig_work = orig_work; @@ -111,7 +97,7 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, bool serial_task = context->is_barrier_block || is_queue_serial static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; Acquire(thr, pc, submit_sync); @@ -126,7 +112,7 @@ static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, } static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, - tsan_block_context_t *context) { + block_context_t *context) { uptr submit_sync = (uptr)context; if (context->submitted_synchronously) Release(thr, pc, submit_sync); @@ -142,7 +128,7 @@ static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, static void dispatch_callback_wrap(void *param) { SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); - tsan_block_context_t *context = (tsan_block_context_t *)param; + block_context_t *context = (block_context_t *)param; dispatch_sync_pre_execute(thr, pc, context); @@ -166,13 +152,13 @@ static void invoke_and_release_block(void *param) { Block_release(block); } -#define DISPATCH_INTERCEPT_B(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ dispatch_block_t heap_block = Block_copy(block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -185,7 +171,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, block, &invoke_block, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -194,11 +180,11 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } -#define DISPATCH_INTERCEPT_F(name, barrier) \ +#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t *new_context = \ + block_context_t *new_context = \ AllocContext(thr, pc, q, context, work); \ new_context->is_barrier_block = barrier; \ Release(thr, pc, (uptr)new_context); \ @@ -211,7 +197,7 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ dispatch_function_t work) { \ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ - tsan_block_context_t new_context = { \ + block_context_t new_context = { \ q, context, work, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ @@ -220,18 +206,21 @@ static void invoke_and_release_block(void *param) { Acquire(thr, pc, (uptr)&new_context); \ } +#define DISPATCH_INTERCEPT(name, barrier) \ + DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ + DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ + DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ + DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) + // We wrap dispatch_async, dispatch_sync and friends where we allocate a new // context, which is used to synchronize (we release the context before // submitting, and the callback acquires it before executing the original // callback). -DISPATCH_INTERCEPT_B(dispatch_async, false) -DISPATCH_INTERCEPT_B(dispatch_barrier_async, true) -DISPATCH_INTERCEPT_F(dispatch_async_f, false) -DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) -DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) -DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) -DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) -DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) +DISPATCH_INTERCEPT(dispatch, false) +DISPATCH_INTERCEPT(dispatch_barrier, true) + +DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, + dispatch_queue_t queue, void *context, dispatch_function_t work) TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, dispatch_queue_t queue, dispatch_block_t block) { @@ -239,7 +228,7 @@ TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); Release(thr, pc, (uptr)new_context); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); @@ -324,9 +313,12 @@ TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, return result; } +// Used, but not intercepted. +extern "C" void dispatch_group_enter(dispatch_group_t group); + TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); - // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. + // Acquired in the group notification callback in dispatch_group_notify[_f]. Release(thr, pc, (uptr)group); REAL(dispatch_group_leave)(group); } @@ -336,10 +328,10 @@ TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); dispatch_retain(group); dispatch_group_enter(group); - __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); + __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); WRAP(dispatch_async)(queue, ^(void) { block_copy(); - _Block_release(block_copy); + Block_release(block_copy); WRAP(dispatch_group_leave)(group); dispatch_release(group); }); @@ -358,6 +350,9 @@ TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, }); } +DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) + TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, dispatch_queue_t q, dispatch_block_t block) { SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); @@ -377,7 +372,7 @@ TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, block(); }); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); new_context->is_barrier_block = true; Release(thr, pc, (uptr)new_context); @@ -395,7 +390,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, if (handler == nullptr) return REAL(dispatch_source_set_event_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0 }; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -424,7 +419,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, if (handler == nullptr) return REAL(dispatch_source_set_cancel_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -455,7 +450,7 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, if (handler == nullptr) return REAL(dispatch_source_set_registration_handler)(source, nullptr); dispatch_queue_t q = GetTargetQueueFromSource(source); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, handler, &invoke_block, false, false, false, 0}; dispatch_block_t new_handler = Block_copy(^(void) { new_context.orig_context = handler; // To explicitly capture "handler". @@ -484,34 +479,54 @@ TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, DISPATCH_NOESCAPE void (^block)(size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); - void *parent_to_child_sync = nullptr; - uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; - void *child_to_parent_sync = nullptr; - uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; - Release(thr, pc, parent_to_child_sync_uptr); + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { SCOPED_INTERCEPTOR_RAW(dispatch_apply); - Acquire(thr, pc, parent_to_child_sync_uptr); + Acquire(thr, pc, parent_to_child_sync); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); block(iteration); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Release(thr, pc, child_to_parent_sync_uptr); + Release(thr, pc, child_to_parent_sync); }; SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); REAL(dispatch_apply)(iterations, queue, new_block); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - Acquire(thr, pc, child_to_parent_sync_uptr); + Acquire(thr, pc, child_to_parent_sync); +} + +static void invoke_block_iteration(void *param, size_t iteration) { + auto block = (void (^)(size_t)) param; + block(iteration); } TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, dispatch_queue_t queue, void *context, void (*work)(void *, size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); + + // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch + // implements dispatch_apply in terms of dispatch_apply_f. + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; + + Release(thr, pc, parent_to_child_sync); void (^new_block)(size_t) = ^(size_t iteration) { + SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); + Acquire(thr, pc, parent_to_child_sync); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); work(context, iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, child_to_parent_sync); }; - WRAP(dispatch_apply)(iterations, queue, new_block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Acquire(thr, pc, child_to_parent_sync); } DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) @@ -531,7 +546,7 @@ TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(destructor); SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); - tsan_block_context_t *new_context = + block_context_t *new_context = AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); uptr submit_sync = (uptr)new_context; Release(thr, pc, submit_sync); @@ -546,7 +561,7 @@ typedef void (^cleanup_handler_t)(int error); TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -563,7 +578,7 @@ TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, dispatch_queue_t q, fd_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { new_context.orig_context = ^(void) { @@ -580,7 +595,7 @@ TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -599,7 +614,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, dispatch_data_t data, dispatch_queue_t q, dispatch_io_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; dispatch_io_handler_t new_h = Block_copy(^(bool done, dispatch_data_t data, int error) { @@ -617,7 +632,7 @@ TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, dispatch_block_t barrier) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { nullptr, nullptr, &invoke_block, false, false, false, 0}; new_context.non_queue_sync_object = (uptr)channel; new_context.is_barrier_block = true; @@ -637,7 +652,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -662,7 +677,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -687,7 +702,7 @@ TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, cleanup_handler_t h) { SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); __block dispatch_io_t new_channel = nullptr; - __block tsan_block_context_t new_context = { + __block block_context_t new_context = { q, nullptr, &invoke_block, false, false, false, 0}; cleanup_handler_t new_h = Block_copy(^(int error) { { @@ -722,6 +737,46 @@ TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { return REAL(dispatch_resume)(o); } -} // namespace __tsan +void InitializeLibdispatchInterceptors() { + INTERCEPT_FUNCTION(dispatch_async); + INTERCEPT_FUNCTION(dispatch_async_f); + INTERCEPT_FUNCTION(dispatch_sync); + INTERCEPT_FUNCTION(dispatch_sync_f); + INTERCEPT_FUNCTION(dispatch_barrier_async); + INTERCEPT_FUNCTION(dispatch_barrier_async_f); + INTERCEPT_FUNCTION(dispatch_barrier_sync); + INTERCEPT_FUNCTION(dispatch_barrier_sync_f); + INTERCEPT_FUNCTION(dispatch_after); + INTERCEPT_FUNCTION(dispatch_after_f); + INTERCEPT_FUNCTION(dispatch_once); + INTERCEPT_FUNCTION(dispatch_once_f); + INTERCEPT_FUNCTION(dispatch_semaphore_signal); + INTERCEPT_FUNCTION(dispatch_semaphore_wait); + INTERCEPT_FUNCTION(dispatch_group_wait); + INTERCEPT_FUNCTION(dispatch_group_leave); + INTERCEPT_FUNCTION(dispatch_group_async); + INTERCEPT_FUNCTION(dispatch_group_async_f); + INTERCEPT_FUNCTION(dispatch_group_notify); + INTERCEPT_FUNCTION(dispatch_group_notify_f); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); + INTERCEPT_FUNCTION(dispatch_apply); + INTERCEPT_FUNCTION(dispatch_apply_f); + INTERCEPT_FUNCTION(dispatch_data_create); + INTERCEPT_FUNCTION(dispatch_read); + INTERCEPT_FUNCTION(dispatch_write); + INTERCEPT_FUNCTION(dispatch_io_read); + INTERCEPT_FUNCTION(dispatch_io_write); + INTERCEPT_FUNCTION(dispatch_io_barrier); + INTERCEPT_FUNCTION(dispatch_io_create); + INTERCEPT_FUNCTION(dispatch_io_create_with_path); + INTERCEPT_FUNCTION(dispatch_io_create_with_io); + INTERCEPT_FUNCTION(dispatch_io_close); + INTERCEPT_FUNCTION(dispatch_resume); +} -#endif // SANITIZER_MAC +} // namespace __tsan diff --git a/lib/tsan/rtl/tsan_malloc_mac.cc b/lib/tsan/rtl/tsan_malloc_mac.cc index 3cc30724da85..0b874aecb99c 100644 --- a/lib/tsan/rtl/tsan_malloc_mac.cc +++ b/lib/tsan/rtl/tsan_malloc_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_malloc_mac.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,19 +28,19 @@ using namespace __tsan; void *p = \ user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) #define COMMON_MALLOC_MALLOC(size) \ - if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ + if (in_symbolizer()) return InternalAlloc(size); \ SCOPED_INTERCEPTOR_RAW(malloc, size); \ void *p = user_alloc(thr, pc, size) #define COMMON_MALLOC_REALLOC(ptr, size) \ - if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \ + if (in_symbolizer()) return InternalRealloc(ptr, size); \ SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ void *p = user_realloc(thr, pc, ptr, size) #define COMMON_MALLOC_CALLOC(count, size) \ - if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \ + if (in_symbolizer()) return InternalCalloc(count, size); \ SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ void *p = user_calloc(thr, pc, size, count) #define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ - if (cur_thread()->in_symbolizer) { \ + if (in_symbolizer()) { \ void *p = InternalAlloc(size, nullptr, alignment); \ if (!p) return errno_ENOMEM; \ *memptr = p; \ @@ -50,12 +49,12 @@ using namespace __tsan; SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, alignment, size); \ int res = user_posix_memalign(thr, pc, memptr, alignment, size); #define COMMON_MALLOC_VALLOC(size) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, GetPageSizeCached()); \ SCOPED_INTERCEPTOR_RAW(valloc, size); \ void *p = user_valloc(thr, pc, size) #define COMMON_MALLOC_FREE(ptr) \ - if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ + if (in_symbolizer()) return InternalFree(ptr); \ SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) #define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); @@ -64,6 +63,8 @@ using namespace __tsan; (void)zone_name; \ Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); #define COMMON_MALLOC_NAMESPACE __tsan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0 #include "sanitizer_common/sanitizer_malloc_mac.inc" diff --git a/lib/tsan/rtl/tsan_md5.cc b/lib/tsan/rtl/tsan_md5.cc index 51279c10d17a..bfe0c17bae75 100644 --- a/lib/tsan/rtl/tsan_md5.cc +++ b/lib/tsan/rtl/tsan_md5.cc @@ -1,9 +1,8 @@ //===-- tsan_md5.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -139,6 +138,14 @@ static const void *body(MD5_CTX *ctx, const void *data, ulong_t size) { return ptr; } +#undef F +#undef G +#undef H +#undef I +#undef STEP +#undef SET +#undef GET + void MD5_Init(MD5_CTX *ctx) { ctx->a = 0x67452301; ctx->b = 0xefcdab89; diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc index b160a9736d5d..f4a95d870cab 100644 --- a/lib/tsan/rtl/tsan_mman.cc +++ b/lib/tsan/rtl/tsan_mman.cc @@ -1,9 +1,8 @@ //===-- tsan_mman.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,6 +201,16 @@ void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { return SetErrnoOnNull(p); } +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) { + if (AllocatorMayReturnNull()) + return SetErrnoOnNull(nullptr); + GET_STACK_TRACE_FATAL(thr, pc); + ReportReallocArrayOverflow(size, n, &stack); + } + return user_realloc(thr, pc, p, size * n); +} + void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); ctx->metamap.AllocBlock(thr, pc, p, sz); diff --git a/lib/tsan/rtl/tsan_mman.h b/lib/tsan/rtl/tsan_mman.h index 6042c5c5d96d..467aabdf2b9d 100644 --- a/lib/tsan/rtl/tsan_mman.h +++ b/lib/tsan/rtl/tsan_mman.h @@ -1,9 +1,8 @@ //===-- tsan_mman.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,6 +34,7 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); void *user_alloc(ThreadState *thr, uptr pc, uptr sz); void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr sz, uptr n); void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz); int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, uptr sz); diff --git a/lib/tsan/rtl/tsan_mutex.cc b/lib/tsan/rtl/tsan_mutex.cc index 22afefce3384..bb7531325aed 100644 --- a/lib/tsan/rtl/tsan_mutex.cc +++ b/lib/tsan/rtl/tsan_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_mutex.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutex.h b/lib/tsan/rtl/tsan_mutex.h index 22ee2f33fcc7..80fdc6ed57bb 100644 --- a/lib/tsan/rtl/tsan_mutex.h +++ b/lib/tsan/rtl/tsan_mutex.h @@ -1,9 +1,8 @@ //===-- tsan_mutex.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.cc b/lib/tsan/rtl/tsan_mutexset.cc index 21587770f687..02e5f9da6080 100644 --- a/lib/tsan/rtl/tsan_mutexset.cc +++ b/lib/tsan/rtl/tsan_mutexset.cc @@ -1,9 +1,8 @@ //===-- tsan_mutexset.cc --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_mutexset.h b/lib/tsan/rtl/tsan_mutexset.h index 605c21a9c08f..d63881f40290 100644 --- a/lib/tsan/rtl/tsan_mutexset.h +++ b/lib/tsan/rtl/tsan_mutexset.h @@ -1,9 +1,8 @@ //===-- tsan_mutexset.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_new_delete.cc b/lib/tsan/rtl/tsan_new_delete.cc index 4f52d3d71eb6..4cbdf703ad28 100644 --- a/lib/tsan/rtl/tsan_new_delete.cc +++ b/lib/tsan/rtl/tsan_new_delete.cc @@ -1,9 +1,8 @@ //===-- tsan_new_delete.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,7 +29,7 @@ DECLARE_REAL(void, free, void *ptr) // TODO(alekseys): throw std::bad_alloc instead of dying on OOM. #define OPERATOR_NEW_BODY(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size); \ void *p = 0; \ { \ @@ -45,7 +44,7 @@ DECLARE_REAL(void, free, void *ptr) return p; #define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalAlloc(size, nullptr, (uptr)align); \ void *p = 0; \ { \ @@ -115,7 +114,7 @@ void *operator new[](__sanitizer::uptr size, std::align_val_t align, #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ - if (cur_thread()->in_symbolizer) \ + if (in_symbolizer()) \ return InternalFree(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h index 8303c241809b..0d106c4147c8 100644 --- a/lib/tsan/rtl/tsan_platform.h +++ b/lib/tsan/rtl/tsan_platform.h @@ -1,9 +1,8 @@ //===-- tsan_platform.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1012,6 +1011,7 @@ void FlushShadowMemory(); void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive); int ExtractResolvFDs(void *state, int *fds, int nfd); int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); +uptr ExtractLongJmpSp(uptr *env); void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size); int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc index d2ce60709067..ec8606f65d5c 100644 --- a/lib/tsan/rtl/tsan_platform_linux.cc +++ b/lib/tsan/rtl/tsan_platform_linux.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_linux.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -68,12 +67,25 @@ extern "C" void *__libc_stack_end; void *__libc_stack_end = 0; #endif -#if SANITIZER_LINUX && defined(__aarch64__) -void InitializeGuardPtr() __attribute__((visibility("hidden"))); +#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO +# define INIT_LONGJMP_XOR_KEY 1 +#else +# define INIT_LONGJMP_XOR_KEY 0 +#endif + +#if INIT_LONGJMP_XOR_KEY +#include "interception/interception.h" +// Must be declared outside of other namespaces. +DECLARE_REAL(int, _setjmp, void *env) #endif namespace __tsan { +#if INIT_LONGJMP_XOR_KEY +static void InitializeLongjmpXorKey(); +static uptr longjmp_xor_key; +#endif + #ifdef TSAN_RUNTIME_VMA // Runtime detected VMA size. uptr vmaSize; @@ -249,7 +261,8 @@ void InitializePlatform() { // Go maps shadow memory lazily and works fine with limited address space. // Unlimited stack is not a problem as well, because the executable // is not compiled with -pie. - if (!SANITIZER_GO) { +#if !SANITIZER_GO + { bool reexec = false; // TSan doesn't play well with unlimited stack size (as stack // overlaps with shadow memory). If we detect unlimited stack size, @@ -284,17 +297,16 @@ void InitializePlatform() { CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); reexec = true; } - // Initialize the guard pointer used in {sig}{set,long}jump. - InitializeGuardPtr(); + // Initialize the xor key used in {sig}{set,long}jump. + InitializeLongjmpXorKey(); #endif if (reexec) ReExec(); } -#if !SANITIZER_GO CheckAndProtect(); InitTlsSize(); -#endif +#endif // !SANITIZER_GO } #if !SANITIZER_GO @@ -335,6 +347,83 @@ int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { return res; } +// Reverse operation of libc stack pointer mangling +static uptr UnmangleLongJmpSp(uptr mangled_sp) { +#if defined(__x86_64__) +# if SANITIZER_LINUX + // Reverse of: + // xor %fs:0x30, %rsi + // rol $0x11, %rsi + uptr sp; + asm("ror $0x11, %0 \n" + "xor %%fs:0x30, %0 \n" + : "=r" (sp) + : "0" (mangled_sp)); + return sp; +# else + return mangled_sp; +# endif +#elif defined(__aarch64__) +# if SANITIZER_LINUX + return mangled_sp ^ longjmp_xor_key; +# else + return mangled_sp; +# endif +#elif defined(__powerpc64__) + // Reverse of: + // ld r4, -28696(r13) + // xor r4, r3, r4 + uptr xor_key; + asm("ld %0, -28696(%%r13)" : "=r" (xor_key)); + return mangled_sp ^ xor_key; +#elif defined(__mips__) + return mangled_sp; +#else + #error "Unknown platform" +#endif +} + +#ifdef __powerpc__ +# define LONG_JMP_SP_ENV_SLOT 0 +#elif SANITIZER_FREEBSD +# define LONG_JMP_SP_ENV_SLOT 2 +#elif SANITIZER_NETBSD +# define LONG_JMP_SP_ENV_SLOT 6 +#elif SANITIZER_LINUX +# ifdef __aarch64__ +# define LONG_JMP_SP_ENV_SLOT 13 +# elif defined(__mips64) +# define LONG_JMP_SP_ENV_SLOT 1 +# else +# define LONG_JMP_SP_ENV_SLOT 6 +# endif +#endif + +uptr ExtractLongJmpSp(uptr *env) { + uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; + return UnmangleLongJmpSp(mangled_sp); +} + +#if INIT_LONGJMP_XOR_KEY +// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp +// functions) by XORing them with a random key. For AArch64 it is a global +// variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by +// issuing a setjmp and XORing the SP pointer values to derive the key. +static void InitializeLongjmpXorKey() { + // 1. Call REAL(setjmp), which stores the mangled SP in env. + jmp_buf env; + REAL(_setjmp)(env); + + // 2. Retrieve vanilla/mangled SP. + uptr sp; + asm("mov %0, sp" : "=r" (sp)); + uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT]; + + // 3. xor SPs to obtain key. + longjmp_xor_key = mangled_sp ^ sp; +} +#endif + void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { // Check that the thr object is in tls; const uptr thr_beg = (uptr)thr; @@ -362,7 +451,7 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, pthread_cleanup_pop(0); return res; } -#endif +#endif // !SANITIZER_GO #if !SANITIZER_GO void ReplaceSystemMalloc() { } @@ -402,6 +491,10 @@ ThreadState *cur_thread() { return thr; } +void set_cur_thread(ThreadState *thr) { + *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); +} + void cur_thread_finalize() { __sanitizer_sigset_t emptyset; internal_sigfillset(&emptyset); diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc index 7e3a47387b6d..0c2d2aa9338e 100644 --- a/lib/tsan/rtl/tsan_platform_mac.cc +++ b/lib/tsan/rtl/tsan_platform_mac.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_mac.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -74,22 +73,22 @@ static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { // shadow memory is set up. static uptr main_thread_identity = 0; ALIGNED(64) static char main_thread_state[sizeof(ThreadState)]; +static ThreadState *main_thread_state_loc = (ThreadState *)main_thread_state; -ThreadState **cur_thread_location() { - ThreadState **thread_identity = (ThreadState **)pthread_self(); - return ((uptr)thread_identity == main_thread_identity) ? nullptr - : thread_identity; +static ThreadState **cur_thread_location() { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity || main_thread_identity == 0) + return &main_thread_state_loc; + return (ThreadState **)MemToShadow(thread_identity); } ThreadState *cur_thread() { - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr || main_thread_identity == 0) { - return (ThreadState *)&main_thread_state; - } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - ThreadState *thr = (ThreadState *)SignalSafeGetOrAllocate( - (uptr *)fake_tls, sizeof(ThreadState)); - return thr; + return (ThreadState *)SignalSafeGetOrAllocate( + (uptr *)cur_thread_location(), sizeof(ThreadState)); +} + +void set_cur_thread(ThreadState *thr) { + *cur_thread_location() = thr; } // TODO(kuba.brecka): This is not async-signal-safe. In particular, we call @@ -97,14 +96,13 @@ ThreadState *cur_thread() { // handler will try to access the unmapped ThreadState. void cur_thread_finalize() { ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + if (thr_state_loc == &main_thread_state_loc) { // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to // exit the main thread. Let's keep the main thread's ThreadState. return; } - ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc); - internal_munmap(*fake_tls, sizeof(ThreadState)); - *fake_tls = nullptr; + internal_munmap(*thr_state_loc, sizeof(ThreadState)); + *thr_state_loc = nullptr; } #endif @@ -213,7 +211,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, ThreadState *parent_thread_state = nullptr; // No parent. int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); CHECK_NE(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ true); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { if (thread == pthread_self()) { @@ -240,8 +238,7 @@ void InitializePlatformEarly() { #endif } -static const uptr kPthreadSetjmpXorKeySlot = 0x7; -extern "C" uptr __tsan_darwin_setjmp_xor_key = 0; +static uptr longjmp_xor_key = 0; void InitializePlatform() { DisableCoreDumperIfNecessary(); @@ -256,21 +253,35 @@ void InitializePlatform() { #endif if (GetMacosVersion() >= MACOS_VERSION_MOJAVE) { - __tsan_darwin_setjmp_xor_key = - (uptr)pthread_getspecific(kPthreadSetjmpXorKeySlot); + // Libsystem currently uses a process-global key; this might change. + const unsigned kTLSLongjmpXorKeySlot = 0x7; + longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot); } } +#ifdef __aarch64__ +# define LONG_JMP_SP_ENV_SLOT \ + ((GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? 12 : 13) +#else +# define LONG_JMP_SP_ENV_SLOT 2 +#endif + +uptr ExtractLongJmpSp(uptr *env) { + uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; + uptr sp = mangled_sp ^ longjmp_xor_key; + return sp; +} + #if !SANITIZER_GO void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { // The pointer to the ThreadState object is stored in the shadow memory // of the tls. uptr tls_end = tls_addr + tls_size; - ThreadState **thr_state_loc = cur_thread_location(); - if (thr_state_loc == nullptr) { + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity) { MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, tls_size); } else { - uptr thr_state_start = (uptr)thr_state_loc; + uptr thr_state_start = thread_identity; uptr thr_state_end = thr_state_start + sizeof(uptr); CHECK_GE(thr_state_start, tls_addr); CHECK_LE(thr_state_start, tls_addr + tls_size); diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc index c38dcc7f2c1c..3bd0f1bd48d8 100644 --- a/lib/tsan/rtl/tsan_platform_posix.cc +++ b/lib/tsan/rtl/tsan_platform_posix.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_posix.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_platform_windows.cc b/lib/tsan/rtl/tsan_platform_windows.cc index 08aa588a4c89..037297559ee0 100644 --- a/lib/tsan/rtl/tsan_platform_windows.cc +++ b/lib/tsan/rtl/tsan_platform_windows.cc @@ -1,9 +1,8 @@ //===-- tsan_platform_windows.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_preinit.cc b/lib/tsan/rtl/tsan_preinit.cc index a96618d7dc81..052b353091d1 100644 --- a/lib/tsan/rtl/tsan_preinit.cc +++ b/lib/tsan/rtl/tsan_preinit.cc @@ -1,9 +1,8 @@ //===-- tsan_preinit.cc ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_report.cc b/lib/tsan/rtl/tsan_report.cc index 629c3e9337ab..ae669024a879 100644 --- a/lib/tsan/rtl/tsan_report.cc +++ b/lib/tsan/rtl/tsan_report.cc @@ -1,9 +1,8 @@ //===-- tsan_report.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -258,7 +257,7 @@ static void PrintThread(const ReportThread *rt) { Printf(" '%s'", rt->name); char thrbuf[kThreadBufSize]; const char *thread_status = rt->running ? "running" : "finished"; - if (rt->workerthread) { + if (rt->thread_type == ThreadType::Worker) { Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status); Printf("\n"); Printf("%s", d.Default()); diff --git a/lib/tsan/rtl/tsan_report.h b/lib/tsan/rtl/tsan_report.h index cdc999c6af1d..b4e4d8989379 100644 --- a/lib/tsan/rtl/tsan_report.h +++ b/lib/tsan/rtl/tsan_report.h @@ -1,9 +1,8 @@ //===-- tsan_report.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #define TSAN_REPORT_H #include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_vector.h" #include "tsan_defs.h" @@ -92,7 +92,7 @@ struct ReportThread { int id; tid_t os_id; bool running; - bool workerthread; + ThreadType thread_type; char *name; u32 parent_tid; ReportStack *stack; diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc index f038e9682d89..8a2704ff0631 100644 --- a/lib/tsan/rtl/tsan_rtl.cc +++ b/lib/tsan/rtl/tsan_rtl.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl.cc -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -329,11 +328,8 @@ static void CheckShadowMapping() { #if !SANITIZER_GO static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - uptr top = 0; - uptr bottom = 0; - bool fast = common_flags()->fast_unwind_on_fatal; - if (fast) GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast); + stack->Unwind(sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) { @@ -354,10 +350,11 @@ void Initialize(ThreadState *thr) { SetCheckFailedCallback(TsanCheckFailed); ctx = new(ctx_placeholder) Context; - const char *options = GetEnv(SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS"); + const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS"; + const char *options = GetEnv(env_name); CacheBinaryName(); CheckASLR(); - InitializeFlags(&ctx->flags, options); + InitializeFlags(&ctx->flags, options, env_name); AvoidCVE_2016_2143(); __sanitizer::InitializePlatformEarly(); __tsan::InitializePlatformEarly(); @@ -397,7 +394,7 @@ void Initialize(ThreadState *thr) { // Initialize thread 0. int tid = ThreadCreate(thr, 0, 0, true); CHECK_EQ(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); #if TSAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif @@ -642,6 +639,7 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, // __m128i _mm_move_epi64(__m128i*); // _mm_storel_epi64(u64*, __m128i); u64 store_word = cur.raw(); + bool stored = false; // scan all the shadow values and dispatch to 4 categories: // same, replace, candidate and race (see comments below). @@ -666,16 +664,28 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, int idx = 0; #include "tsan_update_shadow_word_inl.h" idx = 1; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 2; + if (stored) { +#include "tsan_update_shadow_word_inl.h" + } else { #include "tsan_update_shadow_word_inl.h" + } idx = 3; + if (stored) { #include "tsan_update_shadow_word_inl.h" + } else { +#include "tsan_update_shadow_word_inl.h" + } #endif // we did not find any races and had already stored // the current access info, so we are done - if (LIKELY(store_word == 0)) + if (LIKELY(stored)) return; // choose a random candidate slot and replace it StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); @@ -815,7 +825,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } #endif - if (!SANITIZER_GO && *shadow_mem == kShadowRodata) { + if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. StatInc(thr, StatMop); @@ -826,7 +836,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) { + if (UNLIKELY(fast_state.GetIgnoreBit())) { StatInc(thr, StatMop); StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h index 3410be294502..3a8231bda9a9 100644 --- a/lib/tsan/rtl/tsan_rtl.h +++ b/lib/tsan/rtl/tsan_rtl.h @@ -1,9 +1,8 @@ //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,19 +55,14 @@ namespace __tsan { #if !SANITIZER_GO struct MapUnmapCallback; #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) -static const uptr kAllocatorRegionSizeLog = 20; -static const uptr kAllocatorNumRegions = - SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog; -using ByteMap = TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, - LocalAddressSpaceView, MapUnmapCallback>; + struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = kAllocatorRegionSizeLog; + static const uptr kRegionSizeLog = 20; using AddressSpaceView = LocalAddressSpaceView; - using ByteMap = __tsan::ByteMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -85,10 +79,8 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; #endif -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; Allocator *allocator(); #endif @@ -333,7 +325,6 @@ struct ThreadSignalContext; struct JmpBuf { uptr sp; - uptr mangled_sp; int int_signal_send; bool in_blocking_func; uptr in_signal_handler; @@ -385,6 +376,9 @@ struct ThreadState { // taken by epoch between synchs. // This way we can save one load from tls. u64 fast_synch_epoch; + // Technically `current` should be a separate THREADLOCAL variable; + // but it is placed here in order to share cache line with previous fields. + ThreadState* current; // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. // We do not distinguish beteween ignoring reads and writes // for better performance. @@ -462,12 +456,22 @@ struct ThreadState { #if !SANITIZER_GO #if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); +void set_cur_thread(ThreadState *thr); void cur_thread_finalize(); +INLINE void cur_thread_init() { } #else __attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; INLINE ThreadState *cur_thread() { - return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); + return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current; +} +INLINE void cur_thread_init() { + ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder); + if (UNLIKELY(!thr->current)) + thr->current = thr; +} +INLINE void set_cur_thread(ThreadState *thr) { + reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID @@ -765,7 +769,8 @@ void FuncEntry(ThreadState *thr, uptr pc); void FuncExit(ThreadState *thr); int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread); +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type); void ThreadFinish(ThreadState *thr); int ThreadTid(ThreadState *thr, uptr pc, uptr uid); void ThreadJoin(ThreadState *thr, uptr pc, int tid); @@ -868,6 +873,16 @@ uptr ALWAYS_INLINE HeapEnd() { } #endif +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); +void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); + +// These need to match __tsan_switch_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum FiberSwitchFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + } // namespace __tsan #endif // TSAN_RTL_H diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S index 3d02bf22f8a5..e0b4c71dfed9 100644 --- a/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -3,14 +3,6 @@ #include "sanitizer_common/sanitizer_asm.h" -#if !defined(__APPLE__) -.section .bss -.type __tsan_pointer_chk_guard, %object -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) -__tsan_pointer_chk_guard: -.zero 8 -#endif - #if defined(__APPLE__) .align 2 @@ -40,57 +32,6 @@ _sigsetjmp$non_lazy_ptr: .align 3 #endif -#if !defined(__APPLE__) -// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp -// functions) by XORing them with a random guard pointer. For AArch64 it is a -// global variable rather than a TCB one (as for x86_64/powerpc) and althought -// its value is exported by the loader, it lies within a private GLIBC -// namespace (meaning it should be only used by GLIBC itself and the ABI is -// not stable). So InitializeGuardPtr obtains the pointer guard value by -// issuing a setjmp and checking the resulting pointers values against the -// original ones. -ASM_HIDDEN(_Z18InitializeGuardPtrv) -.global _Z18InitializeGuardPtrv -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) -_Z18InitializeGuardPtrv: - CFI_STARTPROC - // Allocates a jmp_buf for the setjmp call. - stp x29, x30, [sp, -336]! - CFI_DEF_CFA_OFFSET (336) - CFI_OFFSET (29, -336) - CFI_OFFSET (30, -328) - add x29, sp, 0 - CFI_DEF_CFA_REGISTER (29) - add x0, x29, 24 - - // Call libc setjmp that mangle the stack pointer value - adrp x1, :got:_ZN14__interception12real__setjmpE - ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE] - ldr x1, [x1] - blr x1 - - // glibc setjmp mangles both the frame pointer (FP, pc+4 on blr) and the - // stack pointer (SP). FP will be placed on ((uintptr*)jmp_buf)[11] and - // SP at ((uintptr*)jmp_buf)[13]. - // The mangle operation is just 'value' xor 'pointer guard value' and - // if we know the original value (SP) and the expected one, we can derive - // the guard pointer value. - mov x0, sp - - // Loads the mangled SP pointer. - ldr x1, [x29, 128] - eor x0, x0, x1 - adrp x2, __tsan_pointer_chk_guard - str x0, [x2, #:lo12:__tsan_pointer_chk_guard] - ldp x29, x30, [sp], 336 - CFI_RESTORE (30) - CFI_RESTORE (19) - CFI_DEF_CFA (31, 0) - ret - CFI_ENDPROC -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) -#endif - ASM_HIDDEN(__tsan_setjmp) .comm _ZN14__interception11real_setjmpE,8,8 .globl ASM_SYMBOL_INTERCEPTOR(setjmp) @@ -98,7 +39,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) ASM_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -108,33 +49,24 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf - str x19, [sp, 16] - CFI_OFFSET (19, -16) - mov x19, x0 + // Save env parameter + str x0, [sp, 16] + CFI_OFFSET (0, -16) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] - add x0, x29, 32 - eor x1, x2, x0 -#else - adrp x2, ___tsan_darwin_setjmp_xor_key@page - ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - // restore env parameter - mov x0, x19 - ldr x19, [sp, 16] + // Restore env parameter + ldr x0, [sp, 16] + CFI_RESTORE (0) + + // Restore frame/link register ldp x29, x30, [sp], 32 + CFI_RESTORE (29) CFI_RESTORE (30) - CFI_RESTORE (19) CFI_DEF_CFA (31, 0) // tail jump to libc setjmp @@ -158,7 +90,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) ASM_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -168,33 +100,24 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf - str x19, [sp, 16] - CFI_OFFSET (19, -16) - mov x19, x0 + // Save env parameter + str x0, [sp, 16] + CFI_OFFSET (0, -16) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#else - adrp x2, ___tsan_darwin_setjmp_xor_key@page - ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] - add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - // Restore jmp_buf parameter - mov x0, x19 - ldr x19, [sp, 16] + // Restore env parameter + ldr x0, [sp, 16] + CFI_RESTORE (0) + + // Restore frame/link register ldp x29, x30, [sp], 32 + CFI_RESTORE (29) CFI_RESTORE (30) - CFI_RESTORE (19) CFI_DEF_CFA (31, 0) // tail jump to libc setjmp @@ -218,7 +141,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) ASM_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -228,38 +151,26 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf and savesigs - stp x19, x20, [sp, 16] - CFI_OFFSET (19, -16) - CFI_OFFSET (20, -8) - mov w20, w1 - mov x19, x0 + // Save env and savesigs parameter + stp x0, x1, [sp, 16] + CFI_OFFSET (0, -16) + CFI_OFFSET (1, -8) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] - add x0, x29, 32 - eor x1, x2, x0 -#else - adrp x2, ___tsan_darwin_setjmp_xor_key@page - ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - // restore env parameter - mov w1, w20 - mov x0, x19 - ldp x19, x20, [sp, 16] + // Restore env and savesigs parameter + ldp x0, x1, [sp, 16] + CFI_RESTORE (0) + CFI_RESTORE (1) + + // Restore frame/link register ldp x29, x30, [sp], 32 - CFI_RESTORE (30) CFI_RESTORE (29) - CFI_RESTORE (19) - CFI_RESTORE (20) + CFI_RESTORE (30) CFI_DEF_CFA (31, 0) // tail jump to libc sigsetjmp @@ -283,7 +194,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC - // save env parameters for function call + // Save frame/link register stp x29, x30, [sp, -32]! CFI_DEF_CFA_OFFSET (32) CFI_OFFSET (29, -32) @@ -293,32 +204,26 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): add x29, sp, 0 CFI_DEF_CFA_REGISTER (29) - // Save jmp_buf and savesigs - stp x19, x20, [sp, 16] - CFI_OFFSET (19, -16) - CFI_OFFSET (20, -8) - mov w20, w1 - mov x19, x0 + // Save env and savesigs parameter + stp x0, x1, [sp, 16] + CFI_OFFSET (0, -16) + CFI_OFFSET (1, -8) -#if !defined(__APPLE__) - // SP pointer mangling (see glibc setjmp) - adrp x2, __tsan_pointer_chk_guard - ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` add x0, x29, 32 - eor x1, x2, x0 -#endif // call tsan interceptor bl ASM_SYMBOL(__tsan_setjmp) - mov w1, w20 - mov x0, x19 - ldp x19, x20, [sp, 16] + // Restore env and savesigs parameter + ldp x0, x1, [sp, 16] + CFI_RESTORE (0) + CFI_RESTORE (1) + + // Restore frame/link register ldp x29, x30, [sp], 32 - CFI_RESTORE (30) CFI_RESTORE (29) - CFI_RESTORE (19) - CFI_RESTORE (20) + CFI_RESTORE (30) CFI_DEF_CFA (31, 0) // tail jump to libc __sigsetjmp @@ -335,9 +240,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif -#if defined(__linux__) -/* We do not need executable stack. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S index 34ef51c2a725..5913aa360c5d 100644 --- a/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/lib/tsan/rtl/tsan_rtl_amd64.S @@ -189,19 +189,11 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi - mov %rdi, %rsi -#elif defined(__APPLE__) +#elif defined(__linux__) || defined(__APPLE__) lea 16(%rsp), %rdi - mov %rdi, %rsi - xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi -#elif defined(__linux__) - lea 16(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #else # error "Unknown platform" #endif @@ -238,19 +230,11 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi - mov %rdi, %rsi -#elif defined(__APPLE__) - lea 16(%rsp), %rdi - mov %rdi, %rsi - xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi -#elif defined(__linux__) +#elif defined(__linux__) || defined(__APPLE__) lea 16(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #else # error "Unknown platform" #endif @@ -294,19 +278,11 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): // align stack frame sub $8, %rsp CFI_ADJUST_CFA_OFFSET(8) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) || defined(__NetBSD__) lea 24(%rsp), %rdi - mov %rdi, %rsi -#elif defined(__APPLE__) +#elif defined(__linux__) || defined(__APPLE__) lea 32(%rsp), %rdi - mov %rdi, %rsi - xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi -#elif defined(__linux__) - lea 32(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #else # error "Unknown platform" #endif @@ -358,15 +334,11 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): // align stack frame sub $8, %rsp CFI_ADJUST_CFA_OFFSET(8) - // obtain %rsp + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` #if defined(__FreeBSD__) lea 24(%rsp), %rdi - mov %rdi, %rsi #else lea 32(%rsp), %rdi - mov %rdi, %rsi - xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) - rol $0x11, %rsi #endif // call tsan interceptor call ASM_SYMBOL(__tsan_setjmp) @@ -389,10 +361,6 @@ ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif // !defined(__APPLE__) && !defined(__NetBSD__) -#if defined(__FreeBSD__) || defined(__linux__) -/* We do not need executable stack. */ -/* This note is not needed on NetBSD. */ -.section .note.GNU-stack,"",@progbits -#endif +NO_EXEC_STACK_DIRECTIVE #endif diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc index c61d02b7a766..941e70f98872 100644 --- a/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_mutex.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_proc.cc b/lib/tsan/rtl/tsan_rtl_proc.cc index efccdb590ab3..94bbed25b8f5 100644 --- a/lib/tsan/rtl/tsan_rtl_proc.cc +++ b/lib/tsan/rtl/tsan_rtl_proc.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_proc.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc index febb6cef2d39..220a425a2c5b 100644 --- a/lib/tsan/rtl/tsan_rtl_report.cc +++ b/lib/tsan/rtl/tsan_rtl_report.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_report.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,7 +201,7 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { rt->running = (tctx->status == ThreadStatusRunning); rt->name = internal_strdup(tctx->name); rt->parent_tid = tctx->parent_tid; - rt->workerthread = tctx->workerthread; + rt->thread_type = tctx->thread_type; rt->stack = 0; rt->stack = SymbolizeStackId(tctx->creation_stack_id); if (rt->stack) @@ -730,10 +729,12 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) { ALWAYS_INLINE void PrintCurrentStackSlow(uptr pc) { #if !SANITIZER_GO + uptr bp = GET_CURRENT_FRAME(); BufferedStackTrace *ptrace = new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace))) BufferedStackTrace(); - ptrace->Unwind(kStackTraceMax, pc, 0, 0, 0, 0, false); + ptrace->Unwind(pc, bp, nullptr, false); + for (uptr i = 0; i < ptrace->size / 2; i++) { uptr tmp = ptrace->trace_buffer[i]; ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1]; diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc index 766a0f5a505f..fd95cfed4f51 100644 --- a/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/lib/tsan/rtl/tsan_rtl_thread.cc @@ -1,9 +1,8 @@ //===-- tsan_rtl_thread.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -240,13 +239,15 @@ int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { return tid; } -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type) { uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (thread_type != ThreadType::Fiber) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) @@ -258,7 +259,7 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { ThreadRegistry *tr = ctx->thread_registry; OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; - tr->StartThread(tid, os_id, workerthread, &args); + tr->StartThread(tid, os_id, thread_type, &args); tr->Lock(); thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); @@ -404,4 +405,40 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, } } +#if !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) { + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast<ThreadState *>(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, ThreadType::Fiber); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) { + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + internal_free(fiber); +} + +void FiberSwitch(ThreadState *thr, uptr pc, + ThreadState *fiber, unsigned flags) { + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + } // namespace __tsan diff --git a/lib/tsan/rtl/tsan_stack_trace.cc b/lib/tsan/rtl/tsan_stack_trace.cc index a0dee19e246e..dbaca23c68aa 100644 --- a/lib/tsan/rtl/tsan_stack_trace.cc +++ b/lib/tsan/rtl/tsan_stack_trace.cc @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.cc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,3 +48,16 @@ void VarSizeStackTrace::ReverseOrder() { } } // namespace __tsan + +#if !SANITIZER_GO +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + uptr top = 0; + uptr bottom = 0; + if (StackTrace::WillUseFastUnwind(request_fast)) { + GetThreadStackTopAndBottom(false, &top, &bottom); + Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} +#endif // SANITIZER_GO diff --git a/lib/tsan/rtl/tsan_stack_trace.h b/lib/tsan/rtl/tsan_stack_trace.h index f69b57464bcf..3eb8ce156e83 100644 --- a/lib/tsan/rtl/tsan_stack_trace.h +++ b/lib/tsan/rtl/tsan_stack_trace.h @@ -1,9 +1,8 @@ //===-- tsan_stack_trace.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc index 18c83d5c6dac..d23ff47d9af0 100644 --- a/lib/tsan/rtl/tsan_stat.cc +++ b/lib/tsan/rtl/tsan_stat.cc @@ -1,9 +1,8 @@ //===-- tsan_stat.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h index 42d6a2b63532..94e18bc66df9 100644 --- a/lib/tsan/rtl/tsan_stat.h +++ b/lib/tsan/rtl/tsan_stat.h @@ -1,9 +1,8 @@ //===-- tsan_stat.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.cc b/lib/tsan/rtl/tsan_suppressions.cc index 6df074118a10..b3eea9ab5869 100644 --- a/lib/tsan/rtl/tsan_suppressions.cc +++ b/lib/tsan/rtl/tsan_suppressions.cc @@ -1,9 +1,8 @@ //===-- tsan_suppressions.cc ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_suppressions.h b/lib/tsan/rtl/tsan_suppressions.h index 526952d5bd15..f430aeb6c4cf 100644 --- a/lib/tsan/rtl/tsan_suppressions.h +++ b/lib/tsan/rtl/tsan_suppressions.h @@ -1,9 +1,8 @@ //===-- tsan_suppressions.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.cc b/lib/tsan/rtl/tsan_symbolize.cc index 27f0e01c7fbe..cb60763f42f8 100644 --- a/lib/tsan/rtl/tsan_symbolize.cc +++ b/lib/tsan/rtl/tsan_symbolize.cc @@ -1,9 +1,8 @@ //===-- tsan_symbolize.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_symbolize.h b/lib/tsan/rtl/tsan_symbolize.h index 5a9710a3cd30..7adaa04dc273 100644 --- a/lib/tsan/rtl/tsan_symbolize.h +++ b/lib/tsan/rtl/tsan_symbolize.h @@ -1,9 +1,8 @@ //===-- tsan_symbolize.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc index ba3953375ee9..c613b116e3a8 100644 --- a/lib/tsan/rtl/tsan_sync.cc +++ b/lib/tsan/rtl/tsan_sync.cc @@ -1,9 +1,8 @@ //===-- tsan_sync.cc ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_sync.h b/lib/tsan/rtl/tsan_sync.h index 9039970bcf86..47f2739d8de5 100644 --- a/lib/tsan/rtl/tsan_sync.h +++ b/lib/tsan/rtl/tsan_sync.h @@ -1,9 +1,8 @@ //===-- tsan_sync.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_trace.h b/lib/tsan/rtl/tsan_trace.h index 9aef375cbfe6..fbd0f72db6e7 100644 --- a/lib/tsan/rtl/tsan_trace.h +++ b/lib/tsan/rtl/tsan_trace.h @@ -1,9 +1,8 @@ //===-- tsan_trace.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/tsan/rtl/tsan_update_shadow_word_inl.h b/lib/tsan/rtl/tsan_update_shadow_word_inl.h index 6e3ac2fa16f6..056c3aa20320 100644 --- a/lib/tsan/rtl/tsan_update_shadow_word_inl.h +++ b/lib/tsan/rtl/tsan_update_shadow_word_inl.h @@ -1,9 +1,8 @@ //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,31 +17,35 @@ do { const unsigned kAccessSize = 1 << kAccessSizeLog; u64 *sp = &shadow_mem[idx]; old = LoadShadow(sp); - if (old.IsZero()) { + if (LIKELY(old.IsZero())) { StatInc(thr, StatShadowZero); - if (store_word) + if (!stored) { StoreIfNotYetStored(sp, &store_word); - // The above StoreIfNotYetStored could be done unconditionally - // and it even shows 4% gain on synthetic benchmarks (r4307). + stored = true; + } break; } // is the memory access equal to the previous? - if (Shadow::Addr0AndSizeAreEqual(cur, old)) { + if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) { StatInc(thr, StatShadowSameSize); // same thread? - if (Shadow::TidsAreEqual(old, cur)) { + if (LIKELY(Shadow::TidsAreEqual(old, cur))) { StatInc(thr, StatShadowSameThread); - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } StatInc(thr, StatShadowAnotherThread); if (HappensBefore(old, thr)) { - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) { StoreIfNotYetStored(sp, &store_word); + stored = true; + } break; } - if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))) break; goto RACE; } @@ -56,7 +59,7 @@ do { StatInc(thr, StatShadowAnotherThread); if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) break; - if (HappensBefore(old, thr)) + if (LIKELY(HappensBefore(old, thr))) break; goto RACE; } diff --git a/lib/ubsan/ubsan_checks.inc b/lib/ubsan/ubsan_checks.inc index ea82f89e129a..7e7216c5b4ab 100644 --- a/lib/ubsan/ubsan_checks.inc +++ b/lib/ubsan/ubsan_checks.inc @@ -1,9 +1,8 @@ //===-- ubsan_checks.inc ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_diag.cc b/lib/ubsan/ubsan_diag.cc index df4f13cbe2f5..529cc6985763 100644 --- a/lib/ubsan/ubsan_diag.cc +++ b/lib/ubsan/ubsan_diag.cc @@ -1,9 +1,8 @@ //===-- ubsan_diag.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,13 +26,21 @@ using namespace __ubsan; -void __ubsan::GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, - uptr bp, void *context, bool fast) { +// UBSan is combined with runtimes that already provide this functionality +// (e.g., ASan) as well as runtimes that lack it (e.g., scudo). Tried to use +// weak linkage to resolve this issue which is not portable and breaks on +// Windows. +// TODO(yln): This is a temporary workaround. GetStackTrace functions will be +// removed in the future. +void ubsan_GetStackTrace(BufferedStackTrace *stack, uptr max_depth, + uptr pc, uptr bp, void *context, bool fast) { uptr top = 0; uptr bottom = 0; - if (fast) + if (StackTrace::WillUseFastUnwind(fast)) { GetThreadStackTopAndBottom(false, &top, &bottom); - stack->Unwind(max_depth, pc, bp, context, top, bottom, fast); + stack->Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + stack->Unwind(max_depth, pc, bp, context, 0, 0, false); } static void MaybePrintStackTrace(uptr pc, uptr bp) { @@ -43,7 +50,7 @@ static void MaybePrintStackTrace(uptr pc, uptr bp) { return; BufferedStackTrace stack; - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, + ubsan_GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, common_flags()->fast_unwind_on_fatal); stack.Print(); } diff --git a/lib/ubsan/ubsan_diag.h b/lib/ubsan/ubsan_diag.h index bde749684c65..b444e971b228 100644 --- a/lib/ubsan/ubsan_diag.h +++ b/lib/ubsan/ubsan_diag.h @@ -1,9 +1,8 @@ //===-- ubsan_diag.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -235,9 +234,6 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET); GET_CALLER_PC_BP; \ ReportOptions Opts = {unrecoverable_handler, pc, bp} -void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, - void *context, bool fast); - /// \brief Instantiate this class before printing diagnostics in the error /// report. This class ensures that reports from different threads and from /// different sanitizers won't be mixed. diff --git a/lib/ubsan/ubsan_diag_standalone.cc b/lib/ubsan/ubsan_diag_standalone.cc index 1f4a5bd4062b..c22fd1749972 100644 --- a/lib/ubsan/ubsan_diag_standalone.cc +++ b/lib/ubsan/ubsan_diag_standalone.cc @@ -1,9 +1,8 @@ //===-- ubsan_diag_standalone.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,20 +16,23 @@ using namespace __ubsan; -extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_print_stack_trace() { +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { uptr top = 0; uptr bottom = 0; - bool request_fast_unwind = common_flags()->fast_unwind_on_fatal; - if (request_fast_unwind) - __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom); + if (StackTrace::WillUseFastUnwind(request_fast)) { + GetThreadStackTopAndBottom(false, &top, &bottom); + Unwind(max_depth, pc, bp, nullptr, top, bottom, true); + } else + Unwind(max_depth, pc, bp, context, 0, 0, false); +} - GET_CURRENT_PC_BP_SP; - (void)sp; +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_print_stack_trace() { + GET_CURRENT_PC_BP; BufferedStackTrace stack; - stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, - request_fast_unwind); + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal); stack.Print(); } } // extern "C" diff --git a/lib/ubsan/ubsan_flags.cc b/lib/ubsan/ubsan_flags.cc index 7b6784b8278f..8a9498011932 100644 --- a/lib/ubsan/ubsan_flags.cc +++ b/lib/ubsan/ubsan_flags.cc @@ -1,9 +1,8 @@ //===-- ubsan_flags.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -70,7 +69,7 @@ void InitializeFlags() { // Override from user-specified string. parser.ParseString(MaybeCallUbsanDefaultOptions()); // Override from environment variable. - parser.ParseString(GetFlag("UBSAN_OPTIONS")); + parser.ParseStringFromEnv("UBSAN_OPTIONS"); InitializeCommonFlags(); if (Verbosity()) ReportUnrecognizedFlags(); diff --git a/lib/ubsan/ubsan_flags.h b/lib/ubsan/ubsan_flags.h index 18aed9b052ac..daa0d7c701e0 100644 --- a/lib/ubsan/ubsan_flags.h +++ b/lib/ubsan/ubsan_flags.h @@ -1,9 +1,8 @@ //===-- ubsan_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_flags.inc b/lib/ubsan/ubsan_flags.inc index e75a4c44e62c..a4d0e6109e36 100644 --- a/lib/ubsan/ubsan_flags.inc +++ b/lib/ubsan/ubsan_flags.inc @@ -1,9 +1,8 @@ //===-- ubsan_flags.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_handlers.cc b/lib/ubsan/ubsan_handlers.cc index 11e09b0ff0f5..938ac89750f3 100644 --- a/lib/ubsan/ubsan_handlers.cc +++ b/lib/ubsan/ubsan_handlers.cc @@ -1,9 +1,8 @@ //===-- ubsan_handlers.cc -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -599,42 +598,6 @@ void __ubsan::__ubsan_handle_invalid_builtin_abort(InvalidBuiltinData *Data) { Die(); } -static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, - ValueHandle Function, - ReportOptions Opts) { - SourceLocation CallLoc = Data->Loc.acquire(); - ErrorType ET = ErrorType::FunctionTypeMismatch; - - if (ignoreReport(CallLoc, Opts, ET)) - return; - - ScopedReport R(Opts, CallLoc, ET); - - SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); - const char *FName = FLoc.get()->info.function; - if (!FName) - FName = "(unknown)"; - - Diag(CallLoc, DL_Error, ET, - "call to function %0 through pointer to incorrect function type %1") - << FName << Data->Type; - Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; -} - -void -__ubsan::__ubsan_handle_function_type_mismatch(FunctionTypeMismatchData *Data, - ValueHandle Function) { - GET_REPORT_OPTIONS(false); - handleFunctionTypeMismatch(Data, Function, Opts); -} - -void __ubsan::__ubsan_handle_function_type_mismatch_abort( - FunctionTypeMismatchData *Data, ValueHandle Function) { - GET_REPORT_OPTIONS(true); - handleFunctionTypeMismatch(Data, Function, Opts); - Die(); -} - static void handleNonNullReturn(NonNullReturnData *Data, SourceLocation *LocPtr, ReportOptions Opts, bool IsAttr) { if (!LocPtr) diff --git a/lib/ubsan/ubsan_handlers.h b/lib/ubsan/ubsan_handlers.h index 2bf9ff4320e8..22ca96422381 100644 --- a/lib/ubsan/ubsan_handlers.h +++ b/lib/ubsan/ubsan_handlers.h @@ -1,9 +1,8 @@ //===-- ubsan_handlers.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -169,15 +168,6 @@ struct InvalidBuiltinData { /// Handle a builtin called in an invalid way. RECOVERABLE(invalid_builtin, InvalidBuiltinData *Data) -struct FunctionTypeMismatchData { - SourceLocation Loc; - const TypeDescriptor &Type; -}; - -RECOVERABLE(function_type_mismatch, - FunctionTypeMismatchData *Data, - ValueHandle Val) - struct NonNullReturnData { SourceLocation AttrLoc; }; diff --git a/lib/ubsan/ubsan_handlers_cxx.cc b/lib/ubsan/ubsan_handlers_cxx.cc index 85a3e8dad4c5..9c324cc19a11 100644 --- a/lib/ubsan/ubsan_handlers_cxx.cc +++ b/lib/ubsan/ubsan_handlers_cxx.cc @@ -1,9 +1,8 @@ //===-- ubsan_handlers_cxx.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -157,6 +156,50 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, Diag(Loc, DL_Note, ET, "check failed in %0, vtable located in %1") << SrcModule << DstModule; } + +static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, + ValueHandle Function, + ValueHandle calleeRTTI, + ValueHandle fnRTTI, ReportOptions Opts) { + if (checkTypeInfoEquality(reinterpret_cast<void *>(calleeRTTI), + reinterpret_cast<void *>(fnRTTI))) + return false; + + SourceLocation CallLoc = Data->Loc.acquire(); + ErrorType ET = ErrorType::FunctionTypeMismatch; + + if (ignoreReport(CallLoc, Opts, ET)) + return true; + + ScopedReport R(Opts, CallLoc, ET); + + SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); + const char *FName = FLoc.get()->info.function; + if (!FName) + FName = "(unknown)"; + + Diag(CallLoc, DL_Error, ET, + "call to function %0 through pointer to incorrect function type %1") + << FName << Data->Type; + Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; + return true; +} + +void __ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data, + ValueHandle Function, + ValueHandle calleeRTTI, + ValueHandle fnRTTI) { + GET_REPORT_OPTIONS(false); + handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts); +} + +void __ubsan_handle_function_type_mismatch_v1_abort( + FunctionTypeMismatchData *Data, ValueHandle Function, + ValueHandle calleeRTTI, ValueHandle fnRTTI) { + GET_REPORT_OPTIONS(true); + if (handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts)) + Die(); +} } // namespace __ubsan #endif // CAN_SANITIZE_UB diff --git a/lib/ubsan/ubsan_handlers_cxx.h b/lib/ubsan/ubsan_handlers_cxx.h index 2ff014edfcb0..f7b9fc54f472 100644 --- a/lib/ubsan/ubsan_handlers_cxx.h +++ b/lib/ubsan/ubsan_handlers_cxx.h @@ -1,9 +1,8 @@ //===-- ubsan_handlers_cxx.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,6 +33,22 @@ void __ubsan_handle_dynamic_type_cache_miss( extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_dynamic_type_cache_miss_abort( DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash); + +struct FunctionTypeMismatchData { + SourceLocation Loc; + const TypeDescriptor &Type; +}; + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data, + ValueHandle Val, + ValueHandle calleeRTTI, + ValueHandle fnRTTI); +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch_v1_abort(FunctionTypeMismatchData *Data, + ValueHandle Val, + ValueHandle calleeRTTI, + ValueHandle fnRTTI); } #endif // UBSAN_HANDLERS_H diff --git a/lib/ubsan/ubsan_init.cc b/lib/ubsan/ubsan_init.cc index 32fc434ad2bc..f0bbe1ef1076 100644 --- a/lib/ubsan/ubsan_init.cc +++ b/lib/ubsan/ubsan_init.cc @@ -1,9 +1,8 @@ //===-- ubsan_init.cc -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_init.h b/lib/ubsan/ubsan_init.h index f12fc2ced32a..0510385b13dd 100644 --- a/lib/ubsan/ubsan_init.h +++ b/lib/ubsan/ubsan_init.h @@ -1,9 +1,8 @@ //===-- ubsan_init.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_init_standalone.cc b/lib/ubsan/ubsan_init_standalone.cc index 8bd500025cbc..323c2c1f9a47 100644 --- a/lib/ubsan/ubsan_init_standalone.cc +++ b/lib/ubsan/ubsan_init_standalone.cc @@ -1,9 +1,8 @@ //===-- ubsan_init_standalone.cc ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_init_standalone_preinit.cc b/lib/ubsan/ubsan_init_standalone_preinit.cc index 5e75c17ae79a..bf344a2a9fcd 100644 --- a/lib/ubsan/ubsan_init_standalone_preinit.cc +++ b/lib/ubsan/ubsan_init_standalone_preinit.cc @@ -1,9 +1,8 @@ //===-- ubsan_init_standalone_preinit.cc ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_interface.inc b/lib/ubsan/ubsan_interface.inc index 81e06345dcff..1e44bc2171de 100644 --- a/lib/ubsan/ubsan_interface.inc +++ b/lib/ubsan/ubsan_interface.inc @@ -1,9 +1,8 @@ //===-- ubsan_interface.inc -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Ubsan interface list. @@ -22,8 +21,8 @@ INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss) INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort) -INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch) -INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort) +INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1) +INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1_abort) INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion) INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion_abort) INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin) diff --git a/lib/ubsan/ubsan_monitor.cc b/lib/ubsan/ubsan_monitor.cc index e2b39845ce3c..cb97a8ff1b88 100644 --- a/lib/ubsan/ubsan_monitor.cc +++ b/lib/ubsan/ubsan_monitor.cc @@ -1,9 +1,8 @@ //===-- ubsan_monitor.cc ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_monitor.h b/lib/ubsan/ubsan_monitor.h index 7159cbb2c587..3bfd7be89169 100644 --- a/lib/ubsan/ubsan_monitor.h +++ b/lib/ubsan/ubsan_monitor.h @@ -1,9 +1,8 @@ //===-- ubsan_monitor.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_platform.h b/lib/ubsan/ubsan_platform.h index 45a4aa772081..71d7fb18c9b3 100644 --- a/lib/ubsan/ubsan_platform.h +++ b/lib/ubsan/ubsan_platform.h @@ -1,9 +1,8 @@ //===-- ubsan_platform.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_signals_standalone.cc b/lib/ubsan/ubsan_signals_standalone.cc index 5e77c60b1dbb..cc7900cb1364 100644 --- a/lib/ubsan/ubsan_signals_standalone.cc +++ b/lib/ubsan/ubsan_signals_standalone.cc @@ -1,10 +1,9 @@ //=-- ubsan_signals_standalone.cc //------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,11 +38,15 @@ void InitializeDeadlySignals() {} #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #include "sanitizer_common/sanitizer_signal_interceptors.inc" +// TODO(yln): Temporary workaround. Will be removed. +void ubsan_GetStackTrace(BufferedStackTrace *stack, uptr max_depth, + uptr pc, uptr bp, void *context, bool fast); + namespace __ubsan { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + ubsan_GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } diff --git a/lib/ubsan/ubsan_signals_standalone.h b/lib/ubsan/ubsan_signals_standalone.h index b29c29482ec8..128eff266fb7 100644 --- a/lib/ubsan/ubsan_signals_standalone.h +++ b/lib/ubsan/ubsan_signals_standalone.h @@ -1,10 +1,9 @@ //=-- ubsan_signals_standalone.h //------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_type_hash.cc b/lib/ubsan/ubsan_type_hash.cc index a217c862c62a..431495672b55 100644 --- a/lib/ubsan/ubsan_type_hash.cc +++ b/lib/ubsan/ubsan_type_hash.cc @@ -1,9 +1,8 @@ //===-- ubsan_type_hash.cc ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_type_hash.h b/lib/ubsan/ubsan_type_hash.h index aa638713f089..e42884b765a0 100644 --- a/lib/ubsan/ubsan_type_hash.h +++ b/lib/ubsan/ubsan_type_hash.h @@ -1,9 +1,8 @@ //===-- ubsan_type_hash.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -65,6 +64,10 @@ const int VptrMaxOffsetToTop = 1<<20; extern "C" SANITIZER_INTERFACE_ATTRIBUTE HashValue __ubsan_vptr_type_cache[VptrTypeCacheSize]; +/// \brief Do whatever is required by the ABI to check for std::type_info +/// equivalence beyond simple pointer comparison. +bool checkTypeInfoEquality(const void *TypeInfo1, const void *TypeInfo2); + } // namespace __ubsan #endif // UBSAN_TYPE_HASH_H diff --git a/lib/ubsan/ubsan_type_hash_itanium.cc b/lib/ubsan/ubsan_type_hash_itanium.cc index dcce0dd85c26..c4b048f20a8c 100644 --- a/lib/ubsan/ubsan_type_hash_itanium.cc +++ b/lib/ubsan/ubsan_type_hash_itanium.cc @@ -1,9 +1,8 @@ //===-- ubsan_type_hash_itanium.cc ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -118,8 +117,7 @@ static bool isDerivedFromAtOffset(const abi::__class_type_info *Derived, const abi::__class_type_info *Base, sptr Offset) { if (Derived->__type_name == Base->__type_name || - (SANITIZER_NON_UNIQUE_TYPEINFO && - !internal_strcmp(Derived->__type_name, Base->__type_name))) + __ubsan::checkTypeInfoEquality(Derived, Base)) return Offset == 0; if (const abi::__si_class_type_info *SI = @@ -258,4 +256,13 @@ __ubsan::getDynamicTypeInfoFromVtable(void *VtablePtr) { ObjectType ? ObjectType->__type_name : "<unknown>"); } +bool __ubsan::checkTypeInfoEquality(const void *TypeInfo1, + const void *TypeInfo2) { + auto TI1 = static_cast<const std::type_info *>(TypeInfo1); + auto TI2 = static_cast<const std::type_info *>(TypeInfo2); + return SANITIZER_NON_UNIQUE_TYPEINFO && TI1->__type_name[0] != '*' && + TI2->__type_name[0] != '*' && + !internal_strcmp(TI1->__type_name, TI2->__type_name); +} + #endif // CAN_SANITIZE_UB && !SANITIZER_WINDOWS diff --git a/lib/ubsan/ubsan_type_hash_win.cc b/lib/ubsan/ubsan_type_hash_win.cc index 271c4aaf6fa8..c7b2e45af4e6 100644 --- a/lib/ubsan/ubsan_type_hash_win.cc +++ b/lib/ubsan/ubsan_type_hash_win.cc @@ -1,9 +1,8 @@ //===-- ubsan_type_hash_win.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -78,4 +77,8 @@ __ubsan::getDynamicTypeInfoFromVtable(void *VtablePtr) { "<unknown>"); } +bool __ubsan::checkTypeInfoEquality(const void *, const void *) { + return false; +} + #endif // CAN_SANITIZE_UB && SANITIZER_WINDOWS diff --git a/lib/ubsan/ubsan_value.cc b/lib/ubsan/ubsan_value.cc index 466834c09ed1..ba336a6673ca 100644 --- a/lib/ubsan/ubsan_value.cc +++ b/lib/ubsan/ubsan_value.cc @@ -1,9 +1,8 @@ //===-- ubsan_value.cc ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_value.h b/lib/ubsan/ubsan_value.h index 72eee15551ed..a216e3a147e9 100644 --- a/lib/ubsan/ubsan_value.h +++ b/lib/ubsan/ubsan_value.h @@ -1,9 +1,8 @@ //===-- ubsan_value.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_win_dll_thunk.cc b/lib/ubsan/ubsan_win_dll_thunk.cc index a1d0dbd66056..fd39e210af0a 100644 --- a/lib/ubsan/ubsan_win_dll_thunk.cc +++ b/lib/ubsan/ubsan_win_dll_thunk.cc @@ -1,9 +1,8 @@ //===-- ubsan_win_dll_thunk.cc --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc b/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc index c9b74a4c9e09..87ada6131cde 100644 --- a/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc +++ b/lib/ubsan/ubsan_win_dynamic_runtime_thunk.cc @@ -1,9 +1,8 @@ //===-- ubsan_win_dynamic_runtime_thunk.cc --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ubsan/ubsan_win_weak_interception.cc b/lib/ubsan/ubsan_win_weak_interception.cc index c28577057c08..8cf6344ce1d8 100644 --- a/lib/ubsan/ubsan_win_weak_interception.cc +++ b/lib/ubsan/ubsan_win_weak_interception.cc @@ -1,9 +1,8 @@ //===-- ubsan_win_weak_interception.cc ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This module should be included in Ubsan when it is implemented as a shared diff --git a/lib/xray/xray_AArch64.cc b/lib/xray/xray_AArch64.cc index 096de009e83c..4c7805488ab8 100644 --- a/lib/xray/xray_AArch64.cc +++ b/lib/xray/xray_AArch64.cc @@ -1,9 +1,8 @@ //===-- xray_AArch64.cc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_allocator.h b/lib/xray/xray_allocator.h index 907c54542a56..4b42c473261d 100644 --- a/lib/xray/xray_allocator.h +++ b/lib/xray/xray_allocator.h @@ -1,9 +1,8 @@ //===-- xray_allocator.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_arm.cc b/lib/xray/xray_arm.cc index 5b828287e3f6..db26efaa782a 100644 --- a/lib/xray/xray_arm.cc +++ b/lib/xray/xray_arm.cc @@ -1,9 +1,8 @@ //===-- xray_arm.cc ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_flags.cc b/lib/xray/xray_basic_flags.cc index 14d805c71a88..75b674c85656 100644 --- a/lib/xray/xray_basic_flags.cc +++ b/lib/xray/xray_basic_flags.cc @@ -1,9 +1,8 @@ //===-- xray_basic_flags.cc -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_flags.h b/lib/xray/xray_basic_flags.h index 041578f0663c..2459effa8bae 100644 --- a/lib/xray/xray_basic_flags.h +++ b/lib/xray/xray_basic_flags.h @@ -1,9 +1,8 @@ //===-- xray_basic_flags.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_flags.inc b/lib/xray/xray_basic_flags.inc index 327735b51055..fb38c540d356 100644 --- a/lib/xray/xray_basic_flags.inc +++ b/lib/xray/xray_basic_flags.inc @@ -1,9 +1,8 @@ //===-- xray_basic_flags.inc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_logging.cc b/lib/xray/xray_basic_logging.cc index ae1cc0ba79dd..553041ce0c31 100644 --- a/lib/xray/xray_basic_logging.cc +++ b/lib/xray/xray_basic_logging.cc @@ -1,9 +1,8 @@ //===-- xray_basic_logging.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_basic_logging.h b/lib/xray/xray_basic_logging.h index 1639b96d91a1..89caca66b585 100644 --- a/lib/xray/xray_basic_logging.h +++ b/lib/xray/xray_basic_logging.h @@ -1,9 +1,8 @@ //===-- xray_basic_logging.h ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_buffer_queue.cc b/lib/xray/xray_buffer_queue.cc index 7d0e5a1f323c..4cfa717de208 100644 --- a/lib/xray/xray_buffer_queue.cc +++ b/lib/xray/xray_buffer_queue.cc @@ -1,9 +1,8 @@ //===-- xray_buffer_queue.cc -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_buffer_queue.h b/lib/xray/xray_buffer_queue.h index ef2b433f9a3f..e1739d050f3d 100644 --- a/lib/xray/xray_buffer_queue.h +++ b/lib/xray/xray_buffer_queue.h @@ -1,9 +1,8 @@ //===-- xray_buffer_queue.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_defs.h b/lib/xray/xray_defs.h index c009bcc879f1..2da03c3c3451 100644 --- a/lib/xray/xray_defs.h +++ b/lib/xray/xray_defs.h @@ -1,9 +1,8 @@ //===-- xray_defs.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_controller.h b/lib/xray/xray_fdr_controller.h index d44d0309b373..28a3546caa7b 100644 --- a/lib/xray/xray_fdr_controller.h +++ b/lib/xray/xray_fdr_controller.h @@ -1,9 +1,8 @@ //===-- xray_fdr_controller.h ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_flags.cc b/lib/xray/xray_fdr_flags.cc index a14851b1b616..8d432d298d88 100644 --- a/lib/xray/xray_fdr_flags.cc +++ b/lib/xray/xray_fdr_flags.cc @@ -1,9 +1,8 @@ //===-- xray_fdr_flags.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_flags.h b/lib/xray/xray_fdr_flags.h index 9c953f1cabcf..d6f00dc48006 100644 --- a/lib/xray/xray_fdr_flags.h +++ b/lib/xray/xray_fdr_flags.h @@ -1,9 +1,8 @@ //===-- xray_fdr_flags.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_flags.inc b/lib/xray/xray_fdr_flags.inc index d8721ad12cbe..6082b7e78521 100644 --- a/lib/xray/xray_fdr_flags.inc +++ b/lib/xray/xray_fdr_flags.inc @@ -1,9 +1,8 @@ //===-- xray_fdr_flags.inc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_log_records.h b/lib/xray/xray_fdr_log_records.h index e7b1ee562e1b..7a5d438314af 100644 --- a/lib/xray/xray_fdr_log_records.h +++ b/lib/xray/xray_fdr_log_records.h @@ -1,9 +1,8 @@ //===-- xray_fdr_log_records.h -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_log_writer.h b/lib/xray/xray_fdr_log_writer.h index 7712e1377763..0378663c3907 100644 --- a/lib/xray/xray_fdr_log_writer.h +++ b/lib/xray/xray_fdr_log_writer.h @@ -1,9 +1,8 @@ //===-- xray_fdr_log_writer.h ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_logging.cc b/lib/xray/xray_fdr_logging.cc index 1eda26df7a85..abba06576da1 100644 --- a/lib/xray/xray_fdr_logging.cc +++ b/lib/xray/xray_fdr_logging.cc @@ -1,9 +1,8 @@ //===-- xray_fdr_logging.cc ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_fdr_logging.h b/lib/xray/xray_fdr_logging.h index 1639d550a44c..6df0057c4965 100644 --- a/lib/xray/xray_fdr_logging.h +++ b/lib/xray/xray_fdr_logging.h @@ -1,9 +1,8 @@ //===-- xray_fdr_logging.h ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_flags.cc b/lib/xray/xray_flags.cc index b50b68666d80..b9e8324a7874 100644 --- a/lib/xray/xray_flags.cc +++ b/lib/xray/xray_flags.cc @@ -1,9 +1,8 @@ //===-- xray_flags.cc -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -69,7 +68,7 @@ void initializeFlags() XRAY_NEVER_INSTRUMENT { XRayParser.ParseString(XRayCompileFlags); // Override from environment variables. - XRayParser.ParseString(GetEnv("XRAY_OPTIONS")); + XRayParser.ParseStringFromEnv("XRAY_OPTIONS"); // Override from command line. InitializeCommonFlags(); diff --git a/lib/xray/xray_flags.h b/lib/xray/xray_flags.h index 7c1ba9458856..edb5a5119f86 100644 --- a/lib/xray/xray_flags.h +++ b/lib/xray/xray_flags.h @@ -1,9 +1,8 @@ //===-- xray_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_flags.inc b/lib/xray/xray_flags.inc index c87903963a36..b7dc5a08f242 100644 --- a/lib/xray/xray_flags.inc +++ b/lib/xray/xray_flags.inc @@ -1,9 +1,8 @@ //===-- xray_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_function_call_trie.h b/lib/xray/xray_function_call_trie.h index d01ad20e3d71..b8c60583761b 100644 --- a/lib/xray/xray_function_call_trie.h +++ b/lib/xray/xray_function_call_trie.h @@ -1,9 +1,8 @@ //===-- xray_function_call_trie.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_init.cc b/lib/xray/xray_init.cc index b0922aa8e379..b79bc08c5f4d 100644 --- a/lib/xray/xray_init.cc +++ b/lib/xray/xray_init.cc @@ -1,9 +1,8 @@ //===-- xray_init.cc --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_interface.cc b/lib/xray/xray_interface.cc index 6f7b6615b2c0..0d22893eb30f 100644 --- a/lib/xray/xray_interface.cc +++ b/lib/xray/xray_interface.cc @@ -1,9 +1,8 @@ //===-- xray_interface.cpp --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_interface_internal.h b/lib/xray/xray_interface_internal.h index 8ca87457437e..0fea6377648d 100644 --- a/lib/xray/xray_interface_internal.h +++ b/lib/xray/xray_interface_internal.h @@ -1,9 +1,8 @@ //===-- xray_interface_internal.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_log_interface.cc b/lib/xray/xray_log_interface.cc index 0886fd0d1210..7916a9e2b8ad 100644 --- a/lib/xray/xray_log_interface.cc +++ b/lib/xray/xray_log_interface.cc @@ -1,9 +1,8 @@ //===-- xray_log_interface.cc ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_mips.cc b/lib/xray/xray_mips.cc index 6f8243828668..80990ab8d639 100644 --- a/lib/xray/xray_mips.cc +++ b/lib/xray/xray_mips.cc @@ -1,9 +1,8 @@ //===-- xray_mips.cc --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_mips64.cc b/lib/xray/xray_mips64.cc index f1bdf1d7d22d..73c8924f9a0b 100644 --- a/lib/xray/xray_mips64.cc +++ b/lib/xray/xray_mips64.cc @@ -1,9 +1,8 @@ //===-- xray_mips64.cc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_powerpc64.cc b/lib/xray/xray_powerpc64.cc index 5e4938361c0c..abc2becf5b4d 100644 --- a/lib/xray/xray_powerpc64.cc +++ b/lib/xray/xray_powerpc64.cc @@ -1,9 +1,8 @@ //===-- xray_powerpc64.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_powerpc64.inc b/lib/xray/xray_powerpc64.inc index c1a1bac1ad0a..e4e16d5b28e0 100644 --- a/lib/xray/xray_powerpc64.inc +++ b/lib/xray/xray_powerpc64.inc @@ -1,9 +1,8 @@ //===-- xray_powerpc64.inc --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profile_collector.cc b/lib/xray/xray_profile_collector.cc index dc3a82069840..97b52e1d9a22 100644 --- a/lib/xray/xray_profile_collector.cc +++ b/lib/xray/xray_profile_collector.cc @@ -1,9 +1,8 @@ //===-- xray_profile_collector.cc ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profile_collector.h b/lib/xray/xray_profile_collector.h index 86c4ce853797..6e0f252714ba 100644 --- a/lib/xray/xray_profile_collector.h +++ b/lib/xray/xray_profile_collector.h @@ -1,9 +1,8 @@ //===-- xray_profile_collector.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling.cc b/lib/xray/xray_profiling.cc index 4323170cd1bb..66def6cf2485 100644 --- a/lib/xray/xray_profiling.cc +++ b/lib/xray/xray_profiling.cc @@ -1,9 +1,8 @@ //===-- xray_profiling.cc ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling_flags.cc b/lib/xray/xray_profiling_flags.cc index 593e66a78ad2..0e89b7420f8c 100644 --- a/lib/xray/xray_profiling_flags.cc +++ b/lib/xray/xray_profiling_flags.cc @@ -1,9 +1,8 @@ //===-- xray_flags.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling_flags.h b/lib/xray/xray_profiling_flags.h index 2f9a7514799a..d67f240adc88 100644 --- a/lib/xray/xray_profiling_flags.h +++ b/lib/xray/xray_profiling_flags.h @@ -1,9 +1,8 @@ //===-- xray_profiling_flags.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_profiling_flags.inc b/lib/xray/xray_profiling_flags.inc index ccd70860bf61..4f6138872af7 100644 --- a/lib/xray/xray_profiling_flags.inc +++ b/lib/xray/xray_profiling_flags.inc @@ -1,9 +1,8 @@ //===-- xray_profiling_flags.inc --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_recursion_guard.h b/lib/xray/xray_recursion_guard.h index 6edadea563bc..3b6158a2d36c 100644 --- a/lib/xray/xray_recursion_guard.h +++ b/lib/xray/xray_recursion_guard.h @@ -1,9 +1,8 @@ //===-- xray_recursion_guard.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_segmented_array.h b/lib/xray/xray_segmented_array.h index bc7e9379f63b..6eb673edffea 100644 --- a/lib/xray/xray_segmented_array.h +++ b/lib/xray/xray_segmented_array.h @@ -1,9 +1,8 @@ //===-- xray_segmented_array.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_trampoline_mips.S b/lib/xray/xray_trampoline_mips.S index 39a1a3af35c7..499c350d2a24 100644 --- a/lib/xray/xray_trampoline_mips.S +++ b/lib/xray/xray_trampoline_mips.S @@ -1,9 +1,8 @@ //===-- xray_trampoline_mips.s ----------------------------------*- ASM -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_trampoline_mips64.S b/lib/xray/xray_trampoline_mips64.S index 9cbc7e181d9d..d65bec1fc687 100644 --- a/lib/xray/xray_trampoline_mips64.S +++ b/lib/xray/xray_trampoline_mips64.S @@ -1,9 +1,8 @@ //===-- xray_trampoline_mips64.s --------------------------------*- ASM -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_trampoline_x86_64.S b/lib/xray/xray_trampoline_x86_64.S index 52985ffd19ab..1e58362cdc80 100644 --- a/lib/xray/xray_trampoline_x86_64.S +++ b/lib/xray/xray_trampoline_x86_64.S @@ -1,9 +1,8 @@ //===-- xray_trampoline_x86.s -----------------------------------*- ASM -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_tsc.h b/lib/xray/xray_tsc.h index 180d6df188c1..bd7e1911abb3 100644 --- a/lib/xray/xray_tsc.h +++ b/lib/xray/xray_tsc.h @@ -1,9 +1,8 @@ //===-- xray_tsc.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_utils.cc b/lib/xray/xray_utils.cc index 59ba6c3082b2..82674baa5a0c 100644 --- a/lib/xray/xray_utils.cc +++ b/lib/xray/xray_utils.cc @@ -1,9 +1,8 @@ //===-- xray_utils.cc -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -79,7 +78,7 @@ void LogWriter::Flush() XRAY_NEVER_INSTRUMENT { LogWriter *LogWriter::Open() XRAY_NEVER_INSTRUMENT { // Create VMO to hold the profile data. zx_handle_t Vmo; - zx_status_t Status = _zx_vmo_create(0, 0, &Vmo); + zx_status_t Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &Vmo); if (Status != ZX_OK) { Report("XRay: cannot create VMO: %s\n", _zx_status_get_string(Status)); return nullptr; diff --git a/lib/xray/xray_utils.h b/lib/xray/xray_utils.h index 60438973fbd0..333826168c0d 100644 --- a/lib/xray/xray_utils.h +++ b/lib/xray/xray_utils.h @@ -1,9 +1,8 @@ //===-- xray_utils.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/xray/xray_x86_64.inc b/lib/xray/xray_x86_64.inc index b3c475f9110c..477900355cf4 100644 --- a/lib/xray/xray_x86_64.inc +++ b/lib/xray/xray_x86_64.inc @@ -1,9 +1,8 @@ //===-- xray_x86_64.inc -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // |