diff options
Diffstat (limited to 'compiler-rt/lib/scudo/standalone')
34 files changed, 748 insertions, 424 deletions
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h index 62c6f2875106e..3a5aaae736744 100644 --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -14,6 +14,7 @@ #include "flags.h" #include "primary32.h" #include "primary64.h" +#include "secondary.h" #include "size_class_map.h" #include "tsd_exclusive.h" #include "tsd_shared.h" @@ -31,6 +32,7 @@ struct DefaultConfig { // 512KB regions typedef SizeClassAllocator32<SizeClassMap, 19U> Primary; #endif + typedef MapAllocator<> Secondary; template <class A> using TSDRegistryT = TSDRegistryExT<A>; // Exclusive }; @@ -43,6 +45,7 @@ struct AndroidConfig { // 512KB regions typedef SizeClassAllocator32<SizeClassMap, 19U> Primary; #endif + typedef MapAllocator<> Secondary; template <class A> using TSDRegistryT = TSDRegistrySharedT<A, 2U>; // Shared, max 2 TSDs. }; @@ -56,16 +59,20 @@ struct AndroidSvelteConfig { // 64KB regions typedef SizeClassAllocator32<SizeClassMap, 16U> Primary; #endif + typedef MapAllocator<0U> Secondary; template <class A> using TSDRegistryT = TSDRegistrySharedT<A, 1U>; // Shared, only 1 TSD. }; +#if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { // 1GB Regions typedef SizeClassAllocator64<DefaultSizeClassMap, 30U> Primary; + typedef MapAllocator<0U> Secondary; template <class A> using TSDRegistryT = TSDRegistrySharedT<A, 8U>; // Shared, max 8 TSDs. }; +#endif #if SCUDO_ANDROID typedef AndroidConfig Config; diff --git a/compiler-rt/lib/scudo/standalone/atomic_helpers.h b/compiler-rt/lib/scudo/standalone/atomic_helpers.h index 47037d764e252..6c84ba86ed329 100644 --- a/compiler-rt/lib/scudo/standalone/atomic_helpers.h +++ b/compiler-rt/lib/scudo/standalone/atomic_helpers.h @@ -21,12 +21,12 @@ enum memory_order { memory_order_acq_rel = 4, memory_order_seq_cst = 5 }; -COMPILER_CHECK(memory_order_relaxed == __ATOMIC_RELAXED); -COMPILER_CHECK(memory_order_consume == __ATOMIC_CONSUME); -COMPILER_CHECK(memory_order_acquire == __ATOMIC_ACQUIRE); -COMPILER_CHECK(memory_order_release == __ATOMIC_RELEASE); -COMPILER_CHECK(memory_order_acq_rel == __ATOMIC_ACQ_REL); -COMPILER_CHECK(memory_order_seq_cst == __ATOMIC_SEQ_CST); +static_assert(memory_order_relaxed == __ATOMIC_RELAXED, ""); +static_assert(memory_order_consume == __ATOMIC_CONSUME, ""); +static_assert(memory_order_acquire == __ATOMIC_ACQUIRE, ""); +static_assert(memory_order_release == __ATOMIC_RELEASE, ""); +static_assert(memory_order_acq_rel == __ATOMIC_ACQ_REL, ""); +static_assert(memory_order_seq_cst == __ATOMIC_SEQ_CST, ""); struct atomic_u8 { typedef u8 Type; @@ -60,7 +60,7 @@ struct atomic_uptr { }; template <typename T> -INLINE typename T::Type atomic_load(const volatile T *A, memory_order MO) { +inline typename T::Type atomic_load(const volatile T *A, memory_order MO) { DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); typename T::Type V; __atomic_load(&A->ValDoNotUse, &V, MO); @@ -68,29 +68,29 @@ INLINE typename T::Type atomic_load(const volatile T *A, memory_order MO) { } template <typename T> -INLINE void atomic_store(volatile T *A, typename T::Type V, memory_order MO) { +inline void atomic_store(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); __atomic_store(&A->ValDoNotUse, &V, MO); } -INLINE void atomic_thread_fence(memory_order) { __sync_synchronize(); } +inline void atomic_thread_fence(memory_order) { __sync_synchronize(); } template <typename T> -INLINE typename T::Type atomic_fetch_add(volatile T *A, typename T::Type V, +inline typename T::Type atomic_fetch_add(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); return __atomic_fetch_add(&A->ValDoNotUse, V, MO); } template <typename T> -INLINE typename T::Type atomic_fetch_sub(volatile T *A, typename T::Type V, +inline typename T::Type atomic_fetch_sub(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); return __atomic_fetch_sub(&A->ValDoNotUse, V, MO); } template <typename T> -INLINE typename T::Type atomic_exchange(volatile T *A, typename T::Type V, +inline typename T::Type atomic_exchange(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); typename T::Type R; @@ -99,7 +99,7 @@ INLINE typename T::Type atomic_exchange(volatile T *A, typename T::Type V, } template <typename T> -INLINE bool atomic_compare_exchange_strong(volatile T *A, typename T::Type *Cmp, +inline bool atomic_compare_exchange_strong(volatile T *A, typename T::Type *Cmp, typename T::Type Xchg, memory_order MO) { return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, false, MO, @@ -107,7 +107,7 @@ INLINE bool atomic_compare_exchange_strong(volatile T *A, typename T::Type *Cmp, } template <typename T> -INLINE bool atomic_compare_exchange_weak(volatile T *A, typename T::Type *Cmp, +inline bool atomic_compare_exchange_weak(volatile T *A, typename T::Type *Cmp, typename T::Type Xchg, memory_order MO) { return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, true, MO, @@ -117,17 +117,17 @@ INLINE bool atomic_compare_exchange_weak(volatile T *A, typename T::Type *Cmp, // Clutter-reducing helpers. template <typename T> -INLINE typename T::Type atomic_load_relaxed(const volatile T *A) { +inline typename T::Type atomic_load_relaxed(const volatile T *A) { return atomic_load(A, memory_order_relaxed); } template <typename T> -INLINE void atomic_store_relaxed(volatile T *A, typename T::Type V) { +inline void atomic_store_relaxed(volatile T *A, typename T::Type V) { atomic_store(A, V, memory_order_relaxed); } template <typename T> -INLINE typename T::Type atomic_compare_exchange(volatile T *A, +inline typename T::Type atomic_compare_exchange(volatile T *A, typename T::Type Cmp, typename T::Type Xchg) { atomic_compare_exchange_strong(A, &Cmp, Xchg, memory_order_acquire); diff --git a/compiler-rt/lib/scudo/standalone/bytemap.h b/compiler-rt/lib/scudo/standalone/bytemap.h index caeeb2fac879b..a03a0c4710623 100644 --- a/compiler-rt/lib/scudo/standalone/bytemap.h +++ b/compiler-rt/lib/scudo/standalone/bytemap.h @@ -34,6 +34,9 @@ public: return Map[Index]; } + void disable() {} + void enable() {} + private: u8 *Map; }; @@ -82,6 +85,9 @@ public: return Level2Map[Index % Level2Size]; } + void disable() { Mutex.lock(); } + void enable() { Mutex.unlock(); } + private: u8 *get(uptr Index) const { DCHECK_LT(Index, Level1Size); diff --git a/compiler-rt/lib/scudo/standalone/checksum.cpp b/compiler-rt/lib/scudo/standalone/checksum.cpp index f713f5a816093..5de049a0931b2 100644 --- a/compiler-rt/lib/scudo/standalone/checksum.cpp +++ b/compiler-rt/lib/scudo/standalone/checksum.cpp @@ -44,7 +44,6 @@ bool hasHardwareCRC32() { __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); return !!(Ecx & bit_SSE4_2); } - #elif defined(__arm__) || defined(__aarch64__) #ifndef AT_HWCAP #define AT_HWCAP 16 @@ -65,6 +64,9 @@ bool hasHardwareCRC32() { return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); #endif // SCUDO_FUCHSIA } +#else +// No hardware CRC32 implemented in Scudo for other architectures. +bool hasHardwareCRC32() { return false; } #endif // defined(__x86_64__) || defined(__i386__) } // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/checksum.h b/compiler-rt/lib/scudo/standalone/checksum.h index 092342fd6efbd..a63b1b4f064d1 100644 --- a/compiler-rt/lib/scudo/standalone/checksum.h +++ b/compiler-rt/lib/scudo/standalone/checksum.h @@ -37,7 +37,7 @@ enum class Checksum : u8 { // significantly on memory accesses, as well as 1K of CRC32 table, on platforms // that do no support hardware CRC32. The checksum itself is 16-bit, which is at // odds with CRC32, but enough for our needs. -INLINE u16 computeBSDChecksum(u16 Sum, uptr Data) { +inline u16 computeBSDChecksum(u16 Sum, uptr Data) { for (u8 I = 0; I < sizeof(Data); I++) { Sum = static_cast<u16>((Sum >> 1) | ((Sum & 1) << 15)); Sum = static_cast<u16>(Sum + (Data & 0xff)); diff --git a/compiler-rt/lib/scudo/standalone/chunk.h b/compiler-rt/lib/scudo/standalone/chunk.h index 9ae75823ba778..f4d68b3ac6c4a 100644 --- a/compiler-rt/lib/scudo/standalone/chunk.h +++ b/compiler-rt/lib/scudo/standalone/chunk.h @@ -20,7 +20,7 @@ namespace scudo { extern Checksum HashAlgorithm; -INLINE u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) { +inline u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) { // If the hardware CRC32 feature is defined here, it was enabled everywhere, // as opposed to only for crc32_hw.cpp. This means that other hardware // specific instructions were likely emitted at other places, and as a result @@ -71,7 +71,7 @@ struct UnpackedHeader { uptr Checksum : 16; }; typedef atomic_u64 AtomicPackedHeader; -COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader)); +static_assert(sizeof(UnpackedHeader) == sizeof(PackedHeader), ""); // Those constants are required to silence some -Werror=conversion errors when // assigning values to the related bitfield variables. @@ -86,13 +86,12 @@ constexpr uptr getHeaderSize() { return roundUpTo(sizeof(PackedHeader), 1U << SCUDO_MIN_ALIGNMENT_LOG); } -INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) { +inline AtomicPackedHeader *getAtomicHeader(void *Ptr) { return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) - getHeaderSize()); } -INLINE -const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { +inline const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { return reinterpret_cast<const AtomicPackedHeader *>( reinterpret_cast<uptr>(Ptr) - getHeaderSize()); } @@ -100,7 +99,7 @@ const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { // We do not need a cryptographically strong hash for the checksum, but a CRC // type function that can alert us in the event a header is invalid or // corrupted. Ideally slightly better than a simple xor of all fields. -static INLINE u16 computeHeaderChecksum(u32 Cookie, const void *Ptr, +static inline u16 computeHeaderChecksum(u32 Cookie, const void *Ptr, UnpackedHeader *Header) { UnpackedHeader ZeroChecksumHeader = *Header; ZeroChecksumHeader.Checksum = 0; @@ -110,7 +109,7 @@ static INLINE u16 computeHeaderChecksum(u32 Cookie, const void *Ptr, ARRAY_SIZE(HeaderHolder)); } -INLINE void storeHeader(u32 Cookie, void *Ptr, +inline void storeHeader(u32 Cookie, void *Ptr, UnpackedHeader *NewUnpackedHeader) { NewUnpackedHeader->Checksum = computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); @@ -118,9 +117,8 @@ INLINE void storeHeader(u32 Cookie, void *Ptr, atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); } -INLINE -void loadHeader(u32 Cookie, const void *Ptr, - UnpackedHeader *NewUnpackedHeader) { +inline void loadHeader(u32 Cookie, const void *Ptr, + UnpackedHeader *NewUnpackedHeader) { PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader); if (UNLIKELY(NewUnpackedHeader->Checksum != @@ -128,7 +126,7 @@ void loadHeader(u32 Cookie, const void *Ptr, reportHeaderCorruption(const_cast<void *>(Ptr)); } -INLINE void compareExchangeHeader(u32 Cookie, void *Ptr, +inline void compareExchangeHeader(u32 Cookie, void *Ptr, UnpackedHeader *NewUnpackedHeader, UnpackedHeader *OldUnpackedHeader) { NewUnpackedHeader->Checksum = @@ -141,8 +139,8 @@ INLINE void compareExchangeHeader(u32 Cookie, void *Ptr, reportHeaderRace(Ptr); } -INLINE -bool isValid(u32 Cookie, const void *Ptr, UnpackedHeader *NewUnpackedHeader) { +inline bool isValid(u32 Cookie, const void *Ptr, + UnpackedHeader *NewUnpackedHeader) { PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader); return NewUnpackedHeader->Checksum == diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index 60be1dd20d398..a0b4b2973e969 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -18,17 +18,36 @@ #include "quarantine.h" #include "report.h" #include "secondary.h" +#include "string_utils.h" #include "tsd.h" +#ifdef GWP_ASAN_HOOKS +#include "gwp_asan/guarded_pool_allocator.h" +// GWP-ASan is declared here in order to avoid indirect call overhead. It's also +// instantiated outside of the Allocator class, as the allocator is only +// zero-initialised. GWP-ASan requires constant initialisation, and the Scudo +// allocator doesn't have a constexpr constructor (see discussion here: +// https://reviews.llvm.org/D69265#inline-624315). +static gwp_asan::GuardedPoolAllocator GuardedAlloc; +#endif // GWP_ASAN_HOOKS + +extern "C" inline void EmptyCallback() {} + namespace scudo { -template <class Params> class Allocator { +template <class Params, void (*PostInitCallback)(void) = EmptyCallback> +class Allocator { public: using PrimaryT = typename Params::Primary; using CacheT = typename PrimaryT::CacheT; - typedef Allocator<Params> ThisT; + typedef Allocator<Params, PostInitCallback> ThisT; typedef typename Params::template TSDRegistryT<ThisT> TSDRegistryT; + void callPostInitCallback() { + static pthread_once_t OnceControl = PTHREAD_ONCE_INIT; + pthread_once(&OnceControl, PostInitCallback); + } + struct QuarantineCallback { explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache) : Allocator(Instance), Cache(LocalCache) {} @@ -133,6 +152,22 @@ public: Quarantine.init( static_cast<uptr>(getFlags()->quarantine_size_kb << 10), static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10)); + +#ifdef GWP_ASAN_HOOKS + gwp_asan::options::Options Opt; + Opt.Enabled = getFlags()->GWP_ASAN_Enabled; + // Bear in mind - Scudo has its own alignment guarantees that are strictly + // enforced. Scudo exposes the same allocation function for everything from + // malloc() to posix_memalign, so in general this flag goes unused, as Scudo + // will always ask GWP-ASan for an aligned amount of bytes. + Opt.PerfectlyRightAlign = getFlags()->GWP_ASAN_PerfectlyRightAlign; + Opt.MaxSimultaneousAllocations = + getFlags()->GWP_ASAN_MaxSimultaneousAllocations; + Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate; + Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers; + Opt.Printf = Printf; + GuardedAlloc.init(Opt); +#endif // GWP_ASAN_HOOKS } void reset() { memset(this, 0, sizeof(*this)); } @@ -144,7 +179,10 @@ public: TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } - void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); } + // The Cache must be provided zero-initialized. + void initCache(CacheT *Cache) { + Cache->initLinkerInitialized(&Stats, &Primary); + } // Release the resources used by a TSD, which involves: // - draining the local quarantine cache to the global quarantine; @@ -162,6 +200,15 @@ public: bool ZeroContents = false) { initThreadMaybe(); +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.shouldSample())) { + if (void *Ptr = GuardedAlloc.allocate(roundUpTo(Size, Alignment))) + return Ptr; + } +#endif // GWP_ASAN_HOOKS + + ZeroContents |= static_cast<bool>(Options.ZeroContents); + if (UNLIKELY(Alignment > MaxAlignment)) { if (Options.MayReturnNull) return nullptr; @@ -180,12 +227,13 @@ public: ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize()); // Takes care of extravagantly large sizes as well as integer overflows. - if (UNLIKELY(Size >= MaxAllowedMallocSize || - NeededSize >= MaxAllowedMallocSize)) { + static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, ""); + if (UNLIKELY(Size >= MaxAllowedMallocSize)) { if (Options.MayReturnNull) return nullptr; reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize); } + DCHECK_LE(Size, NeededSize); void *Block; uptr ClassId; @@ -200,7 +248,8 @@ public: TSD->unlock(); } else { ClassId = 0; - Block = Secondary.allocate(NeededSize, Alignment, &BlockEnd); + Block = + Secondary.allocate(NeededSize, Alignment, &BlockEnd, ZeroContents); } if (UNLIKELY(!Block)) { @@ -212,22 +261,23 @@ public: // We only need to zero the contents for Primary backed allocations. This // condition is not necessarily unlikely, but since memset is costly, we // might as well mark it as such. - if (UNLIKELY((ZeroContents || Options.ZeroContents) && ClassId)) + if (UNLIKELY(ZeroContents && ClassId)) memset(Block, 0, PrimaryT::getSizeByClassId(ClassId)); + const uptr UnalignedUserPtr = + reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(); + const uptr UserPtr = roundUpTo(UnalignedUserPtr, Alignment); + Chunk::UnpackedHeader Header = {}; - uptr UserPtr = reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(); - if (UNLIKELY(!isAligned(UserPtr, Alignment))) { - const uptr AlignedUserPtr = roundUpTo(UserPtr, Alignment); - const uptr Offset = AlignedUserPtr - UserPtr; - DCHECK_GT(Offset, 2 * sizeof(u32)); + if (UNLIKELY(UnalignedUserPtr != UserPtr)) { + const uptr Offset = UserPtr - UnalignedUserPtr; + DCHECK_GE(Offset, 2 * sizeof(u32)); // The BlockMarker has no security purpose, but is specifically meant for // the chunk iteration function that can be used in debugging situations. // It is the only situation where we have to locate the start of a chunk // based on its block address. reinterpret_cast<u32 *>(Block)[0] = BlockMarker; reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset); - UserPtr = AlignedUserPtr; Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask; } Header.ClassId = ClassId & Chunk::ClassIdMask; @@ -254,6 +304,13 @@ public: // being destroyed properly. Any other heap operation will do a full init. initThreadMaybe(/*MinimalInit=*/true); +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { + GuardedAlloc.deallocate(Ptr); + return; + } +#endif // GWP_ASAN_HOOKS + if (&__scudo_deallocate_hook) __scudo_deallocate_hook(Ptr); @@ -293,6 +350,17 @@ public: DCHECK_NE(OldPtr, nullptr); DCHECK_NE(NewSize, 0); +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) { + uptr OldSize = GuardedAlloc.getSize(OldPtr); + void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment); + if (NewPtr) + memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize); + GuardedAlloc.deallocate(OldPtr); + return NewPtr; + } +#endif // GWP_ASAN_HOOKS + if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment))) reportMisalignedPointer(AllocatorAction::Reallocating, OldPtr); @@ -354,11 +422,14 @@ public: return NewPtr; } - // TODO(kostyak): while this locks the Primary & Secondary, it still allows - // pointers to be fetched from the TSD. We ultimately want to - // lock the registry as well. For now, it's good enough. + // TODO(kostyak): disable() is currently best-effort. There are some small + // windows of time when an allocation could still succeed after + // this function finishes. We will revisit that later. void disable() { initThreadMaybe(); + TSDRegistry.disable(); + Stats.disable(); + Quarantine.disable(); Primary.disable(); Secondary.disable(); } @@ -367,6 +438,9 @@ public: initThreadMaybe(); Secondary.enable(); Primary.enable(); + Quarantine.enable(); + Stats.enable(); + TSDRegistry.enable(); } // The function returns the amount of bytes required to store the statistics, @@ -396,7 +470,10 @@ public: Str.output(); } - void releaseToOS() { Primary.releaseToOS(); } + void releaseToOS() { + initThreadMaybe(); + Primary.releaseToOS(); + } // Iterate over all chunks and call a callback for all busy chunks located // within the provided memory range. Said callback must not use this allocator @@ -409,10 +486,11 @@ public: auto Lambda = [this, From, To, Callback, Arg](uptr Block) { if (Block < From || Block >= To) return; - uptr ChunkSize; - const uptr ChunkBase = getChunkFromBlock(Block, &ChunkSize); - if (ChunkBase != InvalidChunk) - Callback(ChunkBase, ChunkSize, Arg); + uptr Chunk; + Chunk::UnpackedHeader Header; + if (getChunkFromBlock(Block, &Chunk, &Header) && + Header.State == Chunk::State::Allocated) + Callback(Chunk, getSize(reinterpret_cast<void *>(Chunk), &Header), Arg); }; Primary.iterateOverBlocks(Lambda); Secondary.iterateOverBlocks(Lambda); @@ -435,6 +513,12 @@ public: initThreadMaybe(); if (UNLIKELY(!Ptr)) return 0; + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) + return GuardedAlloc.getSize(Ptr); +#endif // GWP_ASAN_HOOKS + Chunk::UnpackedHeader Header; Chunk::loadHeader(Cookie, Ptr, &Header); // Getting the usable size of a chunk only makes sense if it's allocated. @@ -448,8 +532,24 @@ public: Stats.get(S); } + // Returns true if the pointer provided was allocated by the current + // allocator instance, which is compliant with tcmalloc's ownership concept. + // A corrupted chunk will not be reported as owned, which is WAI. + bool isOwned(const void *Ptr) { + initThreadMaybe(); +#ifdef GWP_ASAN_HOOKS + if (GuardedAlloc.pointerIsMine(Ptr)) + return true; +#endif // GWP_ASAN_HOOKS + if (!Ptr || !isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)) + return false; + Chunk::UnpackedHeader Header; + return Chunk::isValid(Cookie, Ptr, &Header) && + Header.State == Chunk::State::Allocated; + } + private: - typedef MapAllocator SecondaryT; + using SecondaryT = typename Params::Secondary; typedef typename PrimaryT::SizeClassMap SizeClassMap; static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG; @@ -459,9 +559,10 @@ private: static const uptr MaxAllowedMallocSize = FIRST_32_SECOND_64(1UL << 31, 1ULL << 40); - // Constants used by the chunk iteration mechanism. + static_assert(MinAlignment >= sizeof(Chunk::PackedHeader), + "Minimal alignment must at least cover a chunk header."); + static const u32 BlockMarker = 0x44554353U; - static const uptr InvalidChunk = ~static_cast<uptr>(0); GlobalStats Stats; TSDRegistryT TSDRegistry; @@ -514,7 +615,7 @@ private: reportSanityCheckError("class ID"); } - static INLINE void *getBlockBegin(const void *Ptr, + static inline void *getBlockBegin(const void *Ptr, Chunk::UnpackedHeader *Header) { return reinterpret_cast<void *>( reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() - @@ -522,7 +623,7 @@ private: } // Return the size of a chunk as requested during its allocation. - INLINE uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) { + inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) { const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; if (LIKELY(Header->ClassId)) return SizeOrUnusedBytes; @@ -539,7 +640,9 @@ private: Chunk::UnpackedHeader NewHeader = *Header; // If the quarantine is disabled, the actual size of a chunk is 0 or larger // than the maximum allowed, we return a chunk directly to the backend. - const bool BypassQuarantine = !Quarantine.getCacheSize() || !Size || + // Logical Or can be short-circuited, which introduces unnecessary + // conditional jumps, so use bitwise Or and let the compiler be clever. + const bool BypassQuarantine = !Quarantine.getCacheSize() | !Size | (Size > Options.QuarantineMaxChunkSize); if (BypassQuarantine) { NewHeader.State = Chunk::State::Available; @@ -567,20 +670,13 @@ private: } } - // This only cares about valid busy chunks. This might change in the future. - uptr getChunkFromBlock(uptr Block, uptr *Size) { + bool getChunkFromBlock(uptr Block, uptr *Chunk, + Chunk::UnpackedHeader *Header) { u32 Offset = 0; if (reinterpret_cast<u32 *>(Block)[0] == BlockMarker) Offset = reinterpret_cast<u32 *>(Block)[1]; - const uptr P = Block + Offset + Chunk::getHeaderSize(); - const void *Ptr = reinterpret_cast<const void *>(P); - Chunk::UnpackedHeader Header; - if (!Chunk::isValid(Cookie, Ptr, &Header) || - Header.State != Chunk::State::Allocated) - return InvalidChunk; - if (Size) - *Size = getSize(Ptr, &Header); - return P; + *Chunk = Block + Offset + Chunk::getHeaderSize(); + return Chunk::isValid(Cookie, reinterpret_cast<void *>(*Chunk), Header); } uptr getStats(ScopedString *Str) { diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h index c015d1ca56696..a76eb6bbc1645 100644 --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -19,22 +19,22 @@ namespace scudo { -template <class Dest, class Source> INLINE Dest bit_cast(const Source &S) { - COMPILER_CHECK(sizeof(Dest) == sizeof(Source)); +template <class Dest, class Source> inline Dest bit_cast(const Source &S) { + static_assert(sizeof(Dest) == sizeof(Source), ""); Dest D; memcpy(&D, &S, sizeof(D)); return D; } -INLINE constexpr uptr roundUpTo(uptr X, uptr Boundary) { +inline constexpr uptr roundUpTo(uptr X, uptr Boundary) { return (X + Boundary - 1) & ~(Boundary - 1); } -INLINE constexpr uptr roundDownTo(uptr X, uptr Boundary) { +inline constexpr uptr roundDownTo(uptr X, uptr Boundary) { return X & ~(Boundary - 1); } -INLINE constexpr bool isAligned(uptr X, uptr Alignment) { +inline constexpr bool isAligned(uptr X, uptr Alignment) { return (X & (Alignment - 1)) == 0; } @@ -48,14 +48,14 @@ template <class T> void Swap(T &A, T &B) { B = Tmp; } -INLINE bool isPowerOfTwo(uptr X) { return (X & (X - 1)) == 0; } +inline bool isPowerOfTwo(uptr X) { return (X & (X - 1)) == 0; } -INLINE uptr getMostSignificantSetBitIndex(uptr X) { +inline uptr getMostSignificantSetBitIndex(uptr X) { DCHECK_NE(X, 0U); return SCUDO_WORDSIZE - 1U - static_cast<uptr>(__builtin_clzl(X)); } -INLINE uptr roundUpToPowerOfTwo(uptr Size) { +inline uptr roundUpToPowerOfTwo(uptr Size) { DCHECK(Size); if (isPowerOfTwo(Size)) return Size; @@ -65,17 +65,17 @@ INLINE uptr roundUpToPowerOfTwo(uptr Size) { return 1UL << (Up + 1); } -INLINE uptr getLeastSignificantSetBitIndex(uptr X) { +inline uptr getLeastSignificantSetBitIndex(uptr X) { DCHECK_NE(X, 0U); return static_cast<uptr>(__builtin_ctzl(X)); } -INLINE uptr getLog2(uptr X) { +inline uptr getLog2(uptr X) { DCHECK(isPowerOfTwo(X)); return getLeastSignificantSetBitIndex(X); } -INLINE u32 getRandomU32(u32 *State) { +inline u32 getRandomU32(u32 *State) { // ANSI C linear congruential PRNG (16-bit output). // return (*State = *State * 1103515245 + 12345) >> 16; // XorShift (32-bit output). @@ -85,11 +85,11 @@ INLINE u32 getRandomU32(u32 *State) { return *State; } -INLINE u32 getRandomModN(u32 *State, u32 N) { +inline u32 getRandomModN(u32 *State, u32 N) { return getRandomU32(State) % N; // [0, N) } -template <typename T> INLINE void shuffle(T *A, u32 N, u32 *RandState) { +template <typename T> inline void shuffle(T *A, u32 N, u32 *RandState) { if (N <= 1) return; u32 State = *RandState; @@ -100,7 +100,7 @@ template <typename T> INLINE void shuffle(T *A, u32 N, u32 *RandState) { // Hardware specific inlinable functions. -INLINE void yieldProcessor(u8 Count) { +inline void yieldProcessor(u8 Count) { #if defined(__i386__) || defined(__x86_64__) __asm__ __volatile__("" ::: "memory"); for (u8 I = 0; I < Count; I++) @@ -117,7 +117,7 @@ INLINE void yieldProcessor(u8 Count) { extern uptr PageSizeCached; uptr getPageSizeSlow(); -INLINE uptr getPageSizeCached() { +inline uptr getPageSizeCached() { // Bionic uses a hardcoded value. if (SCUDO_ANDROID) return 4096U; diff --git a/compiler-rt/lib/scudo/standalone/flags.cpp b/compiler-rt/lib/scudo/standalone/flags.cpp index 1e970ae495056..dd9f050a2d20c 100644 --- a/compiler-rt/lib/scudo/standalone/flags.cpp +++ b/compiler-rt/lib/scudo/standalone/flags.cpp @@ -22,6 +22,13 @@ void Flags::setDefaults() { #define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; #include "flags.inc" #undef SCUDO_FLAG + +#ifdef GWP_ASAN_HOOKS +#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \ + GWP_ASAN_##Name = DefaultValue; +#include "gwp_asan/options.inc" +#undef GWP_ASAN_OPTION +#endif // GWP_ASAN_HOOKS } void registerFlags(FlagParser *Parser, Flags *F) { @@ -30,6 +37,14 @@ void registerFlags(FlagParser *Parser, Flags *F) { reinterpret_cast<void *>(&F->Name)); #include "flags.inc" #undef SCUDO_FLAG + +#ifdef GWP_ASAN_HOOKS +#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \ + Parser->registerFlag("GWP_ASAN_" #Name, Description, FlagType::FT_##Type, \ + reinterpret_cast<void *>(&F->GWP_ASAN_##Name)); +#include "gwp_asan/options.inc" +#undef GWP_ASAN_OPTION +#endif // GWP_ASAN_HOOKS } static const char *getCompileDefinitionScudoDefaultOptions() { diff --git a/compiler-rt/lib/scudo/standalone/flags.h b/compiler-rt/lib/scudo/standalone/flags.h index edd39a1b8ba95..2cd0a5b1334be 100644 --- a/compiler-rt/lib/scudo/standalone/flags.h +++ b/compiler-rt/lib/scudo/standalone/flags.h @@ -17,6 +17,14 @@ struct Flags { #define SCUDO_FLAG(Type, Name, DefaultValue, Description) Type Name; #include "flags.inc" #undef SCUDO_FLAG + +#ifdef GWP_ASAN_HOOKS +#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \ + Type GWP_ASAN_##Name; +#include "gwp_asan/options.inc" +#undef GWP_ASAN_OPTION +#endif // GWP_ASAN_HOOKS + void setDefaults(); }; diff --git a/compiler-rt/lib/scudo/standalone/flags_parser.cpp b/compiler-rt/lib/scudo/standalone/flags_parser.cpp index 070c08b019384..be39fcd4f8879 100644 --- a/compiler-rt/lib/scudo/standalone/flags_parser.cpp +++ b/compiler-rt/lib/scudo/standalone/flags_parser.cpp @@ -108,7 +108,7 @@ void FlagParser::parseString(const char *S) { Pos = OldPos; } -INLINE bool parseBool(const char *Value, bool *b) { +inline bool parseBool(const char *Value, bool *b) { if (strncmp(Value, "0", 1) == 0 || strncmp(Value, "no", 2) == 0 || strncmp(Value, "false", 5) == 0) { *b = false; diff --git a/compiler-rt/lib/scudo/standalone/flags_parser.h b/compiler-rt/lib/scudo/standalone/flags_parser.h index 857b50e880ecc..32511f768c665 100644 --- a/compiler-rt/lib/scudo/standalone/flags_parser.h +++ b/compiler-rt/lib/scudo/standalone/flags_parser.h @@ -29,7 +29,7 @@ public: void printFlagDescriptions(); private: - static const u32 MaxFlags = 12; + static const u32 MaxFlags = 16; struct Flag { const char *Name; const char *Desc; diff --git a/compiler-rt/lib/scudo/standalone/fuchsia.cpp b/compiler-rt/lib/scudo/standalone/fuchsia.cpp index 0a9483ae1dd0d..b3d72de158cf9 100644 --- a/compiler-rt/lib/scudo/standalone/fuchsia.cpp +++ b/compiler-rt/lib/scudo/standalone/fuchsia.cpp @@ -29,7 +29,7 @@ void NORETURN die() { __builtin_trap(); } // We zero-initialize the Extra parameter of map(), make sure this is consistent // with ZX_HANDLE_INVALID. -COMPILER_CHECK(ZX_HANDLE_INVALID == 0); +static_assert(ZX_HANDLE_INVALID == 0, ""); static void *allocateVmar(uptr Size, MapPlatformData *Data, bool AllowNoMem) { // Only scenario so far. @@ -171,7 +171,7 @@ u64 getMonotonicTime() { return _zx_clock_get_monotonic(); } u32 getNumberOfCPUs() { return _zx_system_get_num_cpus(); } bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) { - COMPILER_CHECK(MaxRandomLength <= ZX_CPRNG_DRAW_MAX_LEN); + static_assert(MaxRandomLength <= ZX_CPRNG_DRAW_MAX_LEN, ""); if (UNLIKELY(!Buffer || !Length || Length > MaxRandomLength)) return false; _zx_cprng_draw(Buffer, Length); diff --git a/compiler-rt/lib/scudo/standalone/internal_defs.h b/compiler-rt/lib/scudo/standalone/internal_defs.h index 64ed238ebfec4..8f6a89ecba737 100644 --- a/compiler-rt/lib/scudo/standalone/internal_defs.h +++ b/compiler-rt/lib/scudo/standalone/internal_defs.h @@ -30,7 +30,6 @@ #define INTERFACE __attribute__((visibility("default"))) #define WEAK __attribute__((weak)) -#define INLINE inline #define ALWAYS_INLINE inline __attribute__((always_inline)) #define ALIAS(X) __attribute__((alias(X))) // Please only use the ALIGNED macro before the type. Using ALIGNED after the @@ -84,12 +83,12 @@ void NORETURN reportCheckFailed(const char *File, int Line, #define CHECK_IMPL(C1, Op, C2) \ do { \ - u64 V1 = (u64)(C1); \ - u64 V2 = (u64)(C2); \ + scudo::u64 V1 = (scudo::u64)(C1); \ + scudo::u64 V2 = (scudo::u64)(C2); \ if (UNLIKELY(!(V1 Op V2))) { \ - reportCheckFailed(__FILE__, __LINE__, "(" #C1 ") " #Op " (" #C2 ")", V1, \ - V2); \ - die(); \ + scudo::reportCheckFailed(__FILE__, __LINE__, \ + "(" #C1 ") " #Op " (" #C2 ")", V1, V2); \ + scudo::die(); \ } \ } while (false) @@ -126,8 +125,6 @@ void NORETURN reportCheckFailed(const char *File, int Line, die(); \ } while (0) -#define COMPILER_CHECK(Pred) static_assert(Pred, "") - } // namespace scudo #endif // SCUDO_INTERNAL_DEFS_H_ diff --git a/compiler-rt/lib/scudo/standalone/list.h b/compiler-rt/lib/scudo/standalone/list.h index 6a7b9bd747a71..c3b898a328ca3 100644 --- a/compiler-rt/lib/scudo/standalone/list.h +++ b/compiler-rt/lib/scudo/standalone/list.h @@ -13,43 +13,93 @@ namespace scudo { -// Intrusive POD singly-linked list. +// Intrusive POD singly and doubly linked list. // An object with all zero fields should represent a valid empty list. clear() // should be called on all non-zero-initialized objects before using. -template <class Item> struct IntrusiveList { - friend class Iterator; + +template <class T> class IteratorBase { +public: + explicit IteratorBase(T *CurrentT) : Current(CurrentT) {} + IteratorBase &operator++() { + Current = Current->Next; + return *this; + } + bool operator!=(IteratorBase Other) const { return Current != Other.Current; } + T &operator*() { return *Current; } + +private: + T *Current; +}; + +template <class T> struct IntrusiveList { + bool empty() const { return Size == 0; } + uptr size() const { return Size; } + + T *front() { return First; } + const T *front() const { return First; } + T *back() { return Last; } + const T *back() const { return Last; } void clear() { First = Last = nullptr; Size = 0; } - bool empty() const { return Size == 0; } - uptr size() const { return Size; } + typedef IteratorBase<T> Iterator; + typedef IteratorBase<const T> ConstIterator; - void push_back(Item *X) { - if (empty()) { - X->Next = nullptr; - First = Last = X; - Size = 1; - } else { - X->Next = nullptr; - Last->Next = X; - Last = X; - Size++; + Iterator begin() { return Iterator(First); } + Iterator end() { return Iterator(nullptr); } + + ConstIterator begin() const { return ConstIterator(First); } + ConstIterator end() const { return ConstIterator(nullptr); } + + void checkConsistency() const; + +protected: + uptr Size; + T *First; + T *Last; +}; + +template <class T> void IntrusiveList<T>::checkConsistency() const { + if (Size == 0) { + CHECK_EQ(First, nullptr); + CHECK_EQ(Last, nullptr); + } else { + uptr Count = 0; + for (T *I = First;; I = I->Next) { + Count++; + if (I == Last) + break; } + CHECK_EQ(this->size(), Count); + CHECK_EQ(Last->Next, nullptr); } +} - void push_front(Item *X) { - if (empty()) { - X->Next = nullptr; - First = Last = X; - Size = 1; - } else { - X->Next = First; +template <class T> struct SinglyLinkedList : public IntrusiveList<T> { + using IntrusiveList<T>::First; + using IntrusiveList<T>::Last; + using IntrusiveList<T>::Size; + using IntrusiveList<T>::empty; + + void push_back(T *X) { + X->Next = nullptr; + if (empty()) First = X; - Size++; - } + else + Last->Next = X; + Last = X; + Size++; + } + + void push_front(T *X) { + if (empty()) + Last = X; + X->Next = First; + First = X; + Size++; } void pop_front() { @@ -60,7 +110,7 @@ template <class Item> struct IntrusiveList { Size--; } - void extract(Item *Prev, Item *X) { + void extract(T *Prev, T *X) { DCHECK(!empty()); DCHECK_NE(Prev, nullptr); DCHECK_NE(X, nullptr); @@ -71,84 +121,106 @@ template <class Item> struct IntrusiveList { Size--; } - Item *front() { return First; } - const Item *front() const { return First; } - Item *back() { return Last; } - const Item *back() const { return Last; } - - void append_front(IntrusiveList<Item> *L) { + void append_back(SinglyLinkedList<T> *L) { DCHECK_NE(this, L); if (L->empty()) return; if (empty()) { *this = *L; - } else if (!L->empty()) { - L->Last->Next = First; - First = L->First; + } else { + Last->Next = L->First; + Last = L->Last; Size += L->size(); } L->clear(); } +}; - void append_back(IntrusiveList<Item> *L) { - DCHECK_NE(this, L); - if (L->empty()) - return; +template <class T> struct DoublyLinkedList : IntrusiveList<T> { + using IntrusiveList<T>::First; + using IntrusiveList<T>::Last; + using IntrusiveList<T>::Size; + using IntrusiveList<T>::empty; + + void push_front(T *X) { + X->Prev = nullptr; if (empty()) { - *this = *L; + Last = X; } else { - Last->Next = L->First; - Last = L->Last; - Size += L->size(); + DCHECK_EQ(First->Prev, nullptr); + First->Prev = X; } - L->clear(); + X->Next = First; + First = X; + Size++; + } + + // Inserts X before Y. + void insert(T *X, T *Y) { + if (Y == First) + return push_front(X); + T *Prev = Y->Prev; + // This is a hard CHECK to ensure consistency in the event of an intentional + // corruption of Y->Prev, to prevent a potential write-{4,8}. + CHECK_EQ(Prev->Next, Y); + Prev->Next = X; + X->Prev = Prev; + X->Next = Y; + Y->Prev = X; + Size++; } - void checkConsistency() { - if (Size == 0) { - CHECK_EQ(First, nullptr); - CHECK_EQ(Last, nullptr); + void push_back(T *X) { + X->Next = nullptr; + if (empty()) { + First = X; } else { - uptr Count = 0; - for (Item *I = First;; I = I->Next) { - Count++; - if (I == Last) - break; - } - CHECK_EQ(size(), Count); - CHECK_EQ(Last->Next, nullptr); + DCHECK_EQ(Last->Next, nullptr); + Last->Next = X; } + X->Prev = Last; + Last = X; + Size++; } - template <class ItemT> class IteratorBase { - public: - explicit IteratorBase(ItemT *CurrentItem) : Current(CurrentItem) {} - IteratorBase &operator++() { - Current = Current->Next; - return *this; + void pop_front() { + DCHECK(!empty()); + First = First->Next; + if (!First) + Last = nullptr; + else + First->Prev = nullptr; + Size--; + } + + // The consistency of the adjacent links is aggressively checked in order to + // catch potential corruption attempts, that could yield a mirrored + // write-{4,8} primitive. nullptr checks are deemed less vital. + void remove(T *X) { + T *Prev = X->Prev; + T *Next = X->Next; + if (Prev) { + CHECK_EQ(Prev->Next, X); + Prev->Next = Next; } - bool operator!=(IteratorBase Other) const { - return Current != Other.Current; + if (Next) { + CHECK_EQ(Next->Prev, X); + Next->Prev = Prev; } - ItemT &operator*() { return *Current; } - - private: - ItemT *Current; - }; - - typedef IteratorBase<Item> Iterator; - typedef IteratorBase<const Item> ConstIterator; - - Iterator begin() { return Iterator(First); } - Iterator end() { return Iterator(nullptr); } - - ConstIterator begin() const { return ConstIterator(First); } - ConstIterator end() const { return ConstIterator(nullptr); } - -private: - uptr Size; - Item *First; - Item *Last; + if (First == X) { + DCHECK_EQ(Prev, nullptr); + First = Next; + } else { + DCHECK_NE(Prev, nullptr); + } + if (Last == X) { + DCHECK_EQ(Next, nullptr); + Last = Prev; + } else { + DCHECK_NE(Next, nullptr); + } + Size--; + } }; } // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/platform.h b/compiler-rt/lib/scudo/standalone/platform.h index a897a566f9bf2..a4c2a0b266032 100644 --- a/compiler-rt/lib/scudo/standalone/platform.h +++ b/compiler-rt/lib/scudo/standalone/platform.h @@ -9,13 +9,17 @@ #ifndef SCUDO_PLATFORM_H_ #define SCUDO_PLATFORM_H_ +// Transitive includes of stdint.h specify some of the defines checked below. +#include <stdint.h> + #if defined(__linux__) #define SCUDO_LINUX 1 #else #define SCUDO_LINUX 0 #endif -#if defined(__ANDROID__) +// See https://android.googlesource.com/platform/bionic/+/master/docs/defines.md +#if defined(__BIONIC__) #define SCUDO_ANDROID 1 #else #define SCUDO_ANDROID 0 diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index 9123d07b49b95..e296a78778e00 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -42,7 +42,7 @@ template <class SizeClassMapT, uptr RegionSizeLog> class SizeClassAllocator32 { public: typedef SizeClassMapT SizeClassMap; // Regions should be large enough to hold the largest Block. - COMPILER_CHECK((1UL << RegionSizeLog) >= SizeClassMap::MaxSize); + static_assert((1UL << RegionSizeLog) >= SizeClassMap::MaxSize, ""); typedef SizeClassAllocator32<SizeClassMapT, RegionSizeLog> ThisT; typedef SizeClassAllocatorLocalCache<ThisT> CacheT; typedef typename CacheT::TransferBatch TransferBatch; @@ -123,13 +123,26 @@ public: } void disable() { - for (uptr I = 0; I < NumClasses; I++) - getSizeClassInfo(I)->Mutex.lock(); + // The BatchClassId must be locked last since other classes can use it. + for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) { + if (static_cast<uptr>(I) == SizeClassMap::BatchClassId) + continue; + getSizeClassInfo(static_cast<uptr>(I))->Mutex.lock(); + } + getSizeClassInfo(SizeClassMap::BatchClassId)->Mutex.lock(); + RegionsStashMutex.lock(); + PossibleRegions.disable(); } void enable() { - for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) - getSizeClassInfo(static_cast<uptr>(I))->Mutex.unlock(); + PossibleRegions.enable(); + RegionsStashMutex.unlock(); + getSizeClassInfo(SizeClassMap::BatchClassId)->Mutex.unlock(); + for (uptr I = 0; I < NumClasses; I++) { + if (I == SizeClassMap::BatchClassId) + continue; + getSizeClassInfo(I)->Mutex.unlock(); + } } template <typename F> void iterateOverBlocks(F Callback) { @@ -197,14 +210,14 @@ private: struct ALIGNED(SCUDO_CACHE_LINE_SIZE) SizeClassInfo { HybridMutex Mutex; - IntrusiveList<TransferBatch> FreeList; + SinglyLinkedList<TransferBatch> FreeList; SizeClassStats Stats; bool CanRelease; u32 RandState; uptr AllocatedUser; ReleaseToOsInfo ReleaseInfo; }; - COMPILER_CHECK(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0); + static_assert(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); uptr computeRegionId(uptr Mem) { const uptr Id = Mem >> RegionSizeLog; @@ -300,10 +313,10 @@ private: const uptr NumberOfBlocks = RegionSize / Size; DCHECK_GT(NumberOfBlocks, 0); TransferBatch *B = nullptr; - constexpr uptr ShuffleArraySize = 48; + constexpr u32 ShuffleArraySize = 8U * TransferBatch::MaxNumCached; void *ShuffleArray[ShuffleArraySize]; u32 Count = 0; - const uptr AllocatedUser = NumberOfBlocks * Size; + const uptr AllocatedUser = Size * NumberOfBlocks; for (uptr I = Region; I < Region + AllocatedUser; I += Size) { ShuffleArray[Count++] = reinterpret_cast<void *>(I); if (Count == ShuffleArraySize) { @@ -319,6 +332,11 @@ private: return nullptr; } DCHECK(B); + if (!Sci->FreeList.empty()) { + Sci->FreeList.push_back(B); + B = Sci->FreeList.front(); + Sci->FreeList.pop_front(); + } DCHECK_GT(B->getCount(), 0); C->getStats().add(StatFree, AllocatedUser); @@ -376,7 +394,7 @@ private: for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++) { if (PossibleRegions[I] == ClassId) { ReleaseRecorder Recorder(I * RegionSize); - releaseFreeMemoryToOS(&Sci->FreeList, I * RegionSize, + releaseFreeMemoryToOS(Sci->FreeList, I * RegionSize, RegionSize / PageSize, BlockSize, &Recorder); if (Recorder.getReleasedRangesCount() > 0) { Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index 8f443ea7fa3f3..ef02f0b772d6a 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -125,13 +125,22 @@ public: } void disable() { - for (uptr I = 0; I < NumClasses; I++) - getRegionInfo(I)->Mutex.lock(); + // The BatchClassId must be locked last since other classes can use it. + for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) { + if (static_cast<uptr>(I) == SizeClassMap::BatchClassId) + continue; + getRegionInfo(static_cast<uptr>(I))->Mutex.lock(); + } + getRegionInfo(SizeClassMap::BatchClassId)->Mutex.lock(); } void enable() { - for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) - getRegionInfo(static_cast<uptr>(I))->Mutex.unlock(); + getRegionInfo(SizeClassMap::BatchClassId)->Mutex.unlock(); + for (uptr I = 0; I < NumClasses; I++) { + if (I == SizeClassMap::BatchClassId) + continue; + getRegionInfo(I)->Mutex.unlock(); + } } template <typename F> void iterateOverBlocks(F Callback) const { @@ -187,6 +196,8 @@ private: // Call map for user memory with at least this size. static const uptr MapSizeIncrement = 1UL << 17; + // Fill at most this number of batches from the newly map'd memory. + static const u32 MaxNumBatches = 8U; struct RegionStats { uptr PoppedBlocks; @@ -202,7 +213,7 @@ private: struct ALIGNED(SCUDO_CACHE_LINE_SIZE) RegionInfo { HybridMutex Mutex; - IntrusiveList<TransferBatch> FreeList; + SinglyLinkedList<TransferBatch> FreeList; RegionStats Stats; bool CanRelease; bool Exhausted; @@ -213,7 +224,7 @@ private: MapPlatformData Data; ReleaseToOsInfo ReleaseInfo; }; - COMPILER_CHECK(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0); + static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); uptr PrimaryBase; RegionInfo *RegionInfoArray; @@ -289,16 +300,18 @@ private: C->getStats().add(StatMapped, UserMapSize); } - const uptr NumberOfBlocks = Min( - 8UL * MaxCount, (Region->MappedUser - Region->AllocatedUser) / Size); + const u32 NumberOfBlocks = Min( + MaxNumBatches * MaxCount, + static_cast<u32>((Region->MappedUser - Region->AllocatedUser) / Size)); DCHECK_GT(NumberOfBlocks, 0); TransferBatch *B = nullptr; - constexpr uptr ShuffleArraySize = 48; + constexpr u32 ShuffleArraySize = + MaxNumBatches * TransferBatch::MaxNumCached; void *ShuffleArray[ShuffleArraySize]; u32 Count = 0; const uptr P = RegionBeg + Region->AllocatedUser; - const uptr AllocatedUser = NumberOfBlocks * Size; + const uptr AllocatedUser = Size * NumberOfBlocks; for (uptr I = P; I < P + AllocatedUser; I += Size) { ShuffleArray[Count++] = reinterpret_cast<void *>(I); if (Count == ShuffleArraySize) { @@ -314,6 +327,11 @@ private: return nullptr; } DCHECK(B); + if (!Region->FreeList.empty()) { + Region->FreeList.push_back(B); + B = Region->FreeList.front(); + Region->FreeList.pop_front(); + } DCHECK_GT(B->getCount(), 0); C->getStats().add(StatFree, AllocatedUser); @@ -372,7 +390,7 @@ private: } ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data); - releaseFreeMemoryToOS(&Region->FreeList, Region->RegionBeg, + releaseFreeMemoryToOS(Region->FreeList, Region->RegionBeg, roundUpTo(Region->AllocatedUser, PageSize) / PageSize, BlockSize, &Recorder); diff --git a/compiler-rt/lib/scudo/standalone/quarantine.h b/compiler-rt/lib/scudo/standalone/quarantine.h index 35fd0bc197ea6..406a0e23804d7 100644 --- a/compiler-rt/lib/scudo/standalone/quarantine.h +++ b/compiler-rt/lib/scudo/standalone/quarantine.h @@ -59,7 +59,7 @@ struct QuarantineBatch { void shuffle(u32 State) { ::scudo::shuffle(Batch, Count, &State); } }; -COMPILER_CHECK(sizeof(QuarantineBatch) <= (1U << 13)); // 8Kb. +static_assert(sizeof(QuarantineBatch) <= (1U << 13), ""); // 8Kb. // Per-thread cache of memory blocks. template <typename Callback> class QuarantineCache { @@ -160,7 +160,7 @@ public: } private: - IntrusiveList<QuarantineBatch> List; + SinglyLinkedList<QuarantineBatch> List; atomic_uptr Size; void addToSize(uptr add) { atomic_store_relaxed(&Size, getSize() + add); } @@ -205,7 +205,7 @@ public: ScopedLock L(CacheMutex); Cache.transfer(C); } - if (Cache.getSize() > getMaxSize() && RecyleMutex.tryLock()) + if (Cache.getSize() > getMaxSize() && RecycleMutex.tryLock()) recycle(atomic_load_relaxed(&MinSize), Cb); } @@ -214,7 +214,7 @@ public: ScopedLock L(CacheMutex); Cache.transfer(C); } - RecyleMutex.lock(); + RecycleMutex.lock(); recycle(0, Cb); } @@ -225,11 +225,22 @@ public: getMaxSize() >> 10, getCacheSize() >> 10); } + void disable() { + // RecycleMutex must be locked 1st since we grab CacheMutex within recycle. + RecycleMutex.lock(); + CacheMutex.lock(); + } + + void enable() { + CacheMutex.unlock(); + RecycleMutex.unlock(); + } + private: // Read-only data. alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex CacheMutex; CacheT Cache; - alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex RecyleMutex; + alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex RecycleMutex; atomic_uptr MinSize; atomic_uptr MaxSize; alignas(SCUDO_CACHE_LINE_SIZE) atomic_uptr MaxCacheSize; @@ -261,7 +272,7 @@ private: while (Cache.getSize() > MinSize) Tmp.enqueueBatch(Cache.dequeueBatch()); } - RecyleMutex.unlock(); + RecycleMutex.unlock(); doRecycle(&Tmp, Cb); } diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h index 4fe29fde4bdee..4b5c56ce7c195 100644 --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -149,7 +149,7 @@ private: template <class TransferBatchT, class ReleaseRecorderT> NOINLINE void -releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> *FreeList, uptr Base, +releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base, uptr AllocatedPagesCount, uptr BlockSize, ReleaseRecorderT *Recorder) { const uptr PageSize = getPageSizeCached(); @@ -199,18 +199,18 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> *FreeList, uptr Base, // allocated page. if (BlockSize <= PageSize && PageSize % BlockSize == 0) { // Each chunk affects one page only. - for (auto It = FreeList->begin(); It != FreeList->end(); ++It) { - for (u32 I = 0; I < (*It).getCount(); I++) { - const uptr P = reinterpret_cast<uptr>((*It).get(I)); + for (const auto &It : FreeList) { + for (u32 I = 0; I < It.getCount(); I++) { + const uptr P = reinterpret_cast<uptr>(It.get(I)); if (P >= Base && P < End) Counters.inc((P - Base) >> PageSizeLog); } } } else { // In all other cases chunks might affect more than one page. - for (auto It = FreeList->begin(); It != FreeList->end(); ++It) { - for (u32 I = 0; I < (*It).getCount(); I++) { - const uptr P = reinterpret_cast<uptr>((*It).get(I)); + for (const auto &It : FreeList) { + for (u32 I = 0; I < It.getCount(); I++) { + const uptr P = reinterpret_cast<uptr>(It.get(I)); if (P >= Base && P < End) Counters.incRange((P - Base) >> PageSizeLog, (P - Base + BlockSize - 1) >> PageSizeLog); diff --git a/compiler-rt/lib/scudo/standalone/report.cpp b/compiler-rt/lib/scudo/standalone/report.cpp index 12d851ff019ad..80cc6eda2af92 100644 --- a/compiler-rt/lib/scudo/standalone/report.cpp +++ b/compiler-rt/lib/scudo/standalone/report.cpp @@ -34,7 +34,7 @@ private: ScopedString Message; }; -INLINE void NORETURN trap() { __builtin_trap(); } +inline void NORETURN trap() { __builtin_trap(); } // This could potentially be called recursively if a CHECK fails in the reports. void NORETURN reportCheckFailed(const char *File, int Line, diff --git a/compiler-rt/lib/scudo/standalone/secondary.cpp b/compiler-rt/lib/scudo/standalone/secondary.cpp deleted file mode 100644 index db7361d7134aa..0000000000000 --- a/compiler-rt/lib/scudo/standalone/secondary.cpp +++ /dev/null @@ -1,135 +0,0 @@ -//===-- secondary.cpp -------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "secondary.h" - -#include "string_utils.h" - -namespace scudo { - -// As with the Primary, the size passed to this function includes any desired -// alignment, so that the frontend can align the user allocation. The hint -// parameter allows us to unmap spurious memory when dealing with larger -// (greater than a page) alignments on 32-bit platforms. -// Due to the sparsity of address space available on those platforms, requesting -// an allocation from the Secondary with a large alignment would end up wasting -// VA space (even though we are not committing the whole thing), hence the need -// to trim off some of the reserved space. -// For allocations requested with an alignment greater than or equal to a page, -// the committed memory will amount to something close to Size - AlignmentHint -// (pending rounding and headers). -void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, uptr *BlockEnd) { - DCHECK_GT(Size, AlignmentHint); - const uptr PageSize = getPageSizeCached(); - const uptr MapSize = - roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize) + 2 * PageSize; - MapPlatformData Data = {}; - uptr MapBase = - reinterpret_cast<uptr>(map(nullptr, MapSize, "scudo:secondary", - MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); - if (UNLIKELY(!MapBase)) - return nullptr; - uptr CommitBase = MapBase + PageSize; - uptr MapEnd = MapBase + MapSize; - - // In the unlikely event of alignments larger than a page, adjust the amount - // of memory we want to commit, and trim the extra memory. - if (UNLIKELY(AlignmentHint >= PageSize)) { - // For alignments greater than or equal to a page, the user pointer (eg: the - // pointer that is returned by the C or C++ allocation APIs) ends up on a - // page boundary , and our headers will live in the preceding page. - CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize; - const uptr NewMapBase = CommitBase - PageSize; - DCHECK_GE(NewMapBase, MapBase); - // We only trim the extra memory on 32-bit platforms: 64-bit platforms - // are less constrained memory wise, and that saves us two syscalls. - if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) { - unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data); - MapBase = NewMapBase; - } - const uptr NewMapEnd = CommitBase + PageSize + - roundUpTo((Size - AlignmentHint), PageSize) + - PageSize; - DCHECK_LE(NewMapEnd, MapEnd); - if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) { - unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data); - MapEnd = NewMapEnd; - } - } - - const uptr CommitSize = MapEnd - PageSize - CommitBase; - const uptr Ptr = - reinterpret_cast<uptr>(map(reinterpret_cast<void *>(CommitBase), - CommitSize, "scudo:secondary", 0, &Data)); - LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(Ptr); - H->MapBase = MapBase; - H->MapSize = MapEnd - MapBase; - H->BlockEnd = CommitBase + CommitSize; - H->Data = Data; - { - ScopedLock L(Mutex); - if (LIKELY(Tail)) { - Tail->Next = H; - H->Prev = Tail; - } - Tail = H; - AllocatedBytes += CommitSize; - if (LargestSize < CommitSize) - LargestSize = CommitSize; - NumberOfAllocs++; - Stats.add(StatAllocated, CommitSize); - Stats.add(StatMapped, H->MapSize); - } - if (BlockEnd) - *BlockEnd = CommitBase + CommitSize; - return reinterpret_cast<void *>(Ptr + LargeBlock::getHeaderSize()); -} - -void MapAllocator::deallocate(void *Ptr) { - LargeBlock::Header *H = LargeBlock::getHeader(Ptr); - { - ScopedLock L(Mutex); - LargeBlock::Header *Prev = H->Prev; - LargeBlock::Header *Next = H->Next; - if (Prev) { - CHECK_EQ(Prev->Next, H); - Prev->Next = Next; - } - if (Next) { - CHECK_EQ(Next->Prev, H); - Next->Prev = Prev; - } - if (UNLIKELY(Tail == H)) { - CHECK(!Next); - Tail = Prev; - } else { - CHECK(Next); - } - const uptr CommitSize = H->BlockEnd - reinterpret_cast<uptr>(H); - FreedBytes += CommitSize; - NumberOfFrees++; - Stats.sub(StatAllocated, CommitSize); - Stats.sub(StatMapped, H->MapSize); - } - void *Addr = reinterpret_cast<void *>(H->MapBase); - const uptr Size = H->MapSize; - MapPlatformData Data; - Data = H->Data; - unmap(Addr, Size, UNMAP_ALL, &Data); -} - -void MapAllocator::getStats(ScopedString *Str) const { - Str->append( - "Stats: MapAllocator: allocated %zu times (%zuK), freed %zu times " - "(%zuK), remains %zu (%zuK) max %zuM\n", - NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, - NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10, - LargestSize >> 20); -} - -} // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h index 9d074a57c7722..ab68e5a1d38d7 100644 --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -10,6 +10,7 @@ #define SCUDO_SECONDARY_H_ #include "common.h" +#include "list.h" #include "mutex.h" #include "stats.h" #include "string_utils.h" @@ -47,8 +48,12 @@ static Header *getHeader(const void *Ptr) { } // namespace LargeBlock -class MapAllocator { +template <uptr MaxFreeListSize = 32U> class MapAllocator { public: + // Ensure the freelist is disabled on Fuchsia, since it doesn't support + // releasing Secondary blocks yet. + static_assert(!SCUDO_FUCHSIA || MaxFreeListSize == 0U, ""); + void initLinkerInitialized(GlobalStats *S) { Stats.initLinkerInitialized(); if (LIKELY(S)) @@ -59,7 +64,8 @@ public: initLinkerInitialized(S); } - void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr); + void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr, + bool ZeroContents = false); void deallocate(void *Ptr); @@ -78,13 +84,17 @@ public: void enable() { Mutex.unlock(); } template <typename F> void iterateOverBlocks(F Callback) const { - for (LargeBlock::Header *H = Tail; H != nullptr; H = H->Prev) - Callback(reinterpret_cast<uptr>(H) + LargeBlock::getHeaderSize()); + for (const auto &H : InUseBlocks) + Callback(reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize()); } + static uptr getMaxFreeListSize(void) { return MaxFreeListSize; } + private: HybridMutex Mutex; - LargeBlock::Header *Tail; + DoublyLinkedList<LargeBlock::Header> InUseBlocks; + // The free list is sorted based on the committed size of blocks. + DoublyLinkedList<LargeBlock::Header> FreeBlocks; uptr AllocatedBytes; uptr FreedBytes; uptr LargestSize; @@ -93,6 +103,158 @@ private: LocalStats Stats; }; +// As with the Primary, the size passed to this function includes any desired +// alignment, so that the frontend can align the user allocation. The hint +// parameter allows us to unmap spurious memory when dealing with larger +// (greater than a page) alignments on 32-bit platforms. +// Due to the sparsity of address space available on those platforms, requesting +// an allocation from the Secondary with a large alignment would end up wasting +// VA space (even though we are not committing the whole thing), hence the need +// to trim off some of the reserved space. +// For allocations requested with an alignment greater than or equal to a page, +// the committed memory will amount to something close to Size - AlignmentHint +// (pending rounding and headers). +template <uptr MaxFreeListSize> +void *MapAllocator<MaxFreeListSize>::allocate(uptr Size, uptr AlignmentHint, + uptr *BlockEnd, + bool ZeroContents) { + DCHECK_GE(Size, AlignmentHint); + const uptr PageSize = getPageSizeCached(); + const uptr RoundedSize = + roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize); + + if (MaxFreeListSize && AlignmentHint < PageSize) { + ScopedLock L(Mutex); + for (auto &H : FreeBlocks) { + const uptr FreeBlockSize = H.BlockEnd - reinterpret_cast<uptr>(&H); + if (FreeBlockSize < RoundedSize) + continue; + // Candidate free block should only be at most 4 pages larger. + if (FreeBlockSize > RoundedSize + 4 * PageSize) + break; + FreeBlocks.remove(&H); + InUseBlocks.push_back(&H); + AllocatedBytes += FreeBlockSize; + NumberOfAllocs++; + Stats.add(StatAllocated, FreeBlockSize); + if (BlockEnd) + *BlockEnd = H.BlockEnd; + void *Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(&H) + + LargeBlock::getHeaderSize()); + if (ZeroContents) + memset(Ptr, 0, H.BlockEnd - reinterpret_cast<uptr>(Ptr)); + return Ptr; + } + } + + MapPlatformData Data = {}; + const uptr MapSize = RoundedSize + 2 * PageSize; + uptr MapBase = + reinterpret_cast<uptr>(map(nullptr, MapSize, "scudo:secondary", + MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); + if (UNLIKELY(!MapBase)) + return nullptr; + uptr CommitBase = MapBase + PageSize; + uptr MapEnd = MapBase + MapSize; + + // In the unlikely event of alignments larger than a page, adjust the amount + // of memory we want to commit, and trim the extra memory. + if (UNLIKELY(AlignmentHint >= PageSize)) { + // For alignments greater than or equal to a page, the user pointer (eg: the + // pointer that is returned by the C or C++ allocation APIs) ends up on a + // page boundary , and our headers will live in the preceding page. + CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize; + const uptr NewMapBase = CommitBase - PageSize; + DCHECK_GE(NewMapBase, MapBase); + // We only trim the extra memory on 32-bit platforms: 64-bit platforms + // are less constrained memory wise, and that saves us two syscalls. + if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) { + unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data); + MapBase = NewMapBase; + } + const uptr NewMapEnd = CommitBase + PageSize + + roundUpTo((Size - AlignmentHint), PageSize) + + PageSize; + DCHECK_LE(NewMapEnd, MapEnd); + if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) { + unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data); + MapEnd = NewMapEnd; + } + } + + const uptr CommitSize = MapEnd - PageSize - CommitBase; + const uptr Ptr = + reinterpret_cast<uptr>(map(reinterpret_cast<void *>(CommitBase), + CommitSize, "scudo:secondary", 0, &Data)); + LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(Ptr); + H->MapBase = MapBase; + H->MapSize = MapEnd - MapBase; + H->BlockEnd = CommitBase + CommitSize; + H->Data = Data; + { + ScopedLock L(Mutex); + InUseBlocks.push_back(H); + AllocatedBytes += CommitSize; + if (LargestSize < CommitSize) + LargestSize = CommitSize; + NumberOfAllocs++; + Stats.add(StatAllocated, CommitSize); + Stats.add(StatMapped, H->MapSize); + } + if (BlockEnd) + *BlockEnd = CommitBase + CommitSize; + return reinterpret_cast<void *>(Ptr + LargeBlock::getHeaderSize()); +} + +template <uptr MaxFreeListSize> +void MapAllocator<MaxFreeListSize>::deallocate(void *Ptr) { + LargeBlock::Header *H = LargeBlock::getHeader(Ptr); + const uptr Block = reinterpret_cast<uptr>(H); + { + ScopedLock L(Mutex); + InUseBlocks.remove(H); + const uptr CommitSize = H->BlockEnd - Block; + FreedBytes += CommitSize; + NumberOfFrees++; + Stats.sub(StatAllocated, CommitSize); + if (MaxFreeListSize && FreeBlocks.size() < MaxFreeListSize) { + bool Inserted = false; + for (auto &F : FreeBlocks) { + const uptr FreeBlockSize = F.BlockEnd - reinterpret_cast<uptr>(&F); + if (FreeBlockSize >= CommitSize) { + FreeBlocks.insert(H, &F); + Inserted = true; + break; + } + } + if (!Inserted) + FreeBlocks.push_back(H); + const uptr RoundedAllocationStart = + roundUpTo(Block + LargeBlock::getHeaderSize(), getPageSizeCached()); + MapPlatformData Data = H->Data; + // TODO(kostyak): use release_to_os_interval_ms + releasePagesToOS(Block, RoundedAllocationStart - Block, + H->BlockEnd - RoundedAllocationStart, &Data); + return; + } + Stats.sub(StatMapped, H->MapSize); + } + void *Addr = reinterpret_cast<void *>(H->MapBase); + const uptr Size = H->MapSize; + MapPlatformData Data = H->Data; + unmap(Addr, Size, UNMAP_ALL, &Data); +} + +template <uptr MaxFreeListSize> +void MapAllocator<MaxFreeListSize>::getStats(ScopedString *Str) const { + Str->append( + "Stats: MapAllocator: allocated %zu times (%zuK), freed %zu times " + "(%zuK), remains %zu (%zuK) max %zuM\n", + NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, + NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10, + LargestSize >> 20); +} + } // namespace scudo #endif // SCUDO_SECONDARY_H_ diff --git a/compiler-rt/lib/scudo/standalone/size_class_map.h b/compiler-rt/lib/scudo/standalone/size_class_map.h index dfef0865b9d93..947526e8aea17 100644 --- a/compiler-rt/lib/scudo/standalone/size_class_map.h +++ b/compiler-rt/lib/scudo/standalone/size_class_map.h @@ -49,7 +49,7 @@ public: static const uptr MaxSize = 1UL << MaxSizeLog; static const uptr NumClasses = MidClass + ((MaxSizeLog - MidSizeLog) << S) + 1; - COMPILER_CHECK(NumClasses <= 256); + static_assert(NumClasses <= 256, ""); static const uptr LargestClassId = NumClasses - 1; static const uptr BatchClassId = 0; @@ -120,7 +120,8 @@ public: if (C < LargestClassId) CHECK_EQ(getClassIdBySize(S + 1), C + 1); CHECK_EQ(getClassIdBySize(S - 1), C); - CHECK_GT(getSizeByClassId(C), getSizeByClassId(C - 1)); + if (C - 1 != BatchClassId) + CHECK_GT(getSizeByClassId(C), getSizeByClassId(C - 1)); } // Do not perform the loop if the maximum size is too large. if (MaxSizeLog > 19) @@ -129,7 +130,7 @@ public: const uptr C = getClassIdBySize(S); CHECK_LT(C, NumClasses); CHECK_GE(getSizeByClassId(C), S); - if (C > 0) + if (C - 1 != BatchClassId) CHECK_LT(getSizeByClassId(C - 1), S); } } diff --git a/compiler-rt/lib/scudo/standalone/stats.h b/compiler-rt/lib/scudo/standalone/stats.h index 16ef5b89b854c..38481e98e48d1 100644 --- a/compiler-rt/lib/scudo/standalone/stats.h +++ b/compiler-rt/lib/scudo/standalone/stats.h @@ -10,6 +10,7 @@ #define SCUDO_STATS_H_ #include "atomic_helpers.h" +#include "list.h" #include "mutex.h" #include <string.h> @@ -45,20 +46,17 @@ public: uptr get(StatType I) const { return atomic_load_relaxed(&StatsArray[I]); } -private: - friend class GlobalStats; - atomic_uptr StatsArray[StatCount]; LocalStats *Next; LocalStats *Prev; + +private: + atomic_uptr StatsArray[StatCount]; }; // Global stats, used for aggregation and querying. class GlobalStats : public LocalStats { public: - void initLinkerInitialized() { - Next = this; - Prev = this; - } + void initLinkerInitialized() {} void init() { memset(this, 0, sizeof(*this)); initLinkerInitialized(); @@ -66,38 +64,35 @@ public: void link(LocalStats *S) { ScopedLock L(Mutex); - S->Next = Next; - S->Prev = this; - Next->Prev = S; - Next = S; + StatsList.push_back(S); } void unlink(LocalStats *S) { ScopedLock L(Mutex); - S->Prev->Next = S->Next; - S->Next->Prev = S->Prev; + StatsList.remove(S); for (uptr I = 0; I < StatCount; I++) add(static_cast<StatType>(I), S->get(static_cast<StatType>(I))); } void get(uptr *S) const { - memset(S, 0, StatCount * sizeof(uptr)); ScopedLock L(Mutex); - const LocalStats *Stats = this; - for (;;) { + for (uptr I = 0; I < StatCount; I++) + S[I] = LocalStats::get(static_cast<StatType>(I)); + for (const auto &Stats : StatsList) { for (uptr I = 0; I < StatCount; I++) - S[I] += Stats->get(static_cast<StatType>(I)); - Stats = Stats->Next; - if (Stats == this) - break; + S[I] += Stats.get(static_cast<StatType>(I)); } // All stats must be non-negative. for (uptr I = 0; I < StatCount; I++) S[I] = static_cast<sptr>(S[I]) >= 0 ? S[I] : 0; } + void disable() { Mutex.lock(); } + void enable() { Mutex.unlock(); } + private: mutable HybridMutex Mutex; + DoublyLinkedList<LocalStats> StatsList; }; } // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/tsd.h b/compiler-rt/lib/scudo/standalone/tsd.h index f24ff01960fb2..20f0d69cabfd1 100644 --- a/compiler-rt/lib/scudo/standalone/tsd.h +++ b/compiler-rt/lib/scudo/standalone/tsd.h @@ -14,6 +14,7 @@ #include "mutex.h" #include <limits.h> // for PTHREAD_DESTRUCTOR_ITERATIONS +#include <pthread.h> // With some build setups, this might still not be defined. #ifndef PTHREAD_DESTRUCTOR_ITERATIONS @@ -38,7 +39,7 @@ template <class Allocator> struct ALIGNED(SCUDO_CACHE_LINE_SIZE) TSD { void commitBack(Allocator *Instance) { Instance->commitBack(this); } - INLINE bool tryLock() { + inline bool tryLock() { if (Mutex.tryLock()) { atomic_store_relaxed(&Precedence, 0); return true; @@ -49,12 +50,12 @@ template <class Allocator> struct ALIGNED(SCUDO_CACHE_LINE_SIZE) TSD { static_cast<uptr>(getMonotonicTime() >> FIRST_32_SECOND_64(16, 0))); return false; } - INLINE void lock() { + inline void lock() { atomic_store_relaxed(&Precedence, 0); Mutex.lock(); } - INLINE void unlock() { Mutex.unlock(); } - INLINE uptr getPrecedence() { return atomic_load_relaxed(&Precedence); } + inline void unlock() { Mutex.unlock(); } + inline uptr getPrecedence() { return atomic_load_relaxed(&Precedence); } private: HybridMutex Mutex; diff --git a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h index 971ae4857fcab..69479ea7bdf40 100644 --- a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h +++ b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h @@ -11,8 +11,6 @@ #include "tsd.h" -#include <pthread.h> - namespace scudo { enum class ThreadState : u8 { @@ -48,7 +46,8 @@ template <class Allocator> struct TSDRegistryExT { } ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) { - if (LIKELY(State == ThreadState::Initialized)) { + if (LIKELY(State == ThreadState::Initialized && + !atomic_load(&Disabled, memory_order_acquire))) { *UnlockRequired = false; return &ThreadTSD; } @@ -58,6 +57,20 @@ template <class Allocator> struct TSDRegistryExT { return FallbackTSD; } + // To disable the exclusive TSD registry, we effectively lock the fallback TSD + // and force all threads to attempt to use it instead of their local one. + void disable() { + Mutex.lock(); + FallbackTSD->lock(); + atomic_store(&Disabled, 1U, memory_order_release); + } + + void enable() { + atomic_store(&Disabled, 0U, memory_order_release); + FallbackTSD->unlock(); + Mutex.unlock(); + } + private: void initOnceMaybe(Allocator *Instance) { ScopedLock L(Mutex); @@ -77,10 +90,12 @@ private: pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0); ThreadTSD.initLinkerInitialized(Instance); State = ThreadState::Initialized; + Instance->callPostInitCallback(); } pthread_key_t PThreadKey; bool Initialized; + atomic_u8 Disabled; TSD<Allocator> *FallbackTSD; HybridMutex Mutex; static THREADLOCAL ThreadState State; diff --git a/compiler-rt/lib/scudo/standalone/tsd_shared.h b/compiler-rt/lib/scudo/standalone/tsd_shared.h index da88a897b8f52..5ab8269519a90 100644 --- a/compiler-rt/lib/scudo/standalone/tsd_shared.h +++ b/compiler-rt/lib/scudo/standalone/tsd_shared.h @@ -12,8 +12,6 @@ #include "linux.h" // for getAndroidTlsPtr() #include "tsd.h" -#include <pthread.h> - namespace scudo { template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT { @@ -50,6 +48,8 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT { void unmapTestOnly() { unmap(reinterpret_cast<void *>(TSDs), sizeof(TSD<Allocator>) * NumberOfTSDs); + setCurrentTSD(nullptr); + pthread_key_delete(PThreadKey); } ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, @@ -70,9 +70,21 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT { return getTSDAndLockSlow(TSD); } + void disable() { + Mutex.lock(); + for (u32 I = 0; I < NumberOfTSDs; I++) + TSDs[I].lock(); + } + + void enable() { + for (s32 I = NumberOfTSDs - 1; I >= 0; I--) + TSDs[I].unlock(); + Mutex.unlock(); + } + private: ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) { -#if SCUDO_ANDROID +#if _BIONIC *getAndroidTlsPtr() = reinterpret_cast<uptr>(CurrentTSD); #elif SCUDO_LINUX ThreadTSD = CurrentTSD; @@ -84,7 +96,7 @@ private: } ALWAYS_INLINE TSD<Allocator> *getCurrentTSD() { -#if SCUDO_ANDROID +#if _BIONIC return reinterpret_cast<TSD<Allocator> *>(*getAndroidTlsPtr()); #elif SCUDO_LINUX return ThreadTSD; @@ -105,6 +117,7 @@ private: // Initial context assignment is done in a plain round-robin fashion. const u32 Index = atomic_fetch_add(&CurrentIndex, 1U, memory_order_relaxed); setCurrentTSD(&TSDs[Index % NumberOfTSDs]); + Instance->callPostInitCallback(); } NOINLINE TSD<Allocator> *getTSDAndLockSlow(TSD<Allocator> *CurrentTSD) { @@ -152,12 +165,12 @@ private: u32 CoPrimes[MaxTSDCount]; bool Initialized; HybridMutex Mutex; -#if SCUDO_LINUX && !SCUDO_ANDROID +#if SCUDO_LINUX && !_BIONIC static THREADLOCAL TSD<Allocator> *ThreadTSD; #endif }; -#if SCUDO_LINUX && !SCUDO_ANDROID +#if SCUDO_LINUX && !_BIONIC template <class Allocator, u32 MaxTSDCount> THREADLOCAL TSD<Allocator> *TSDRegistrySharedT<Allocator, MaxTSDCount>::ThreadTSD; diff --git a/compiler-rt/lib/scudo/standalone/vector.h b/compiler-rt/lib/scudo/standalone/vector.h index 3cb4005ed29cc..6ca350a257713 100644 --- a/compiler-rt/lib/scudo/standalone/vector.h +++ b/compiler-rt/lib/scudo/standalone/vector.h @@ -84,7 +84,8 @@ private: DCHECK_LE(Size, NewCapacity); const uptr NewCapacityBytes = roundUpTo(NewCapacity * sizeof(T), getPageSizeCached()); - T *NewData = (T *)map(nullptr, NewCapacityBytes, "scudo:vector"); + T *NewData = + reinterpret_cast<T *>(map(nullptr, NewCapacityBytes, "scudo:vector")); if (Data) { memcpy(NewData, Data, Size * sizeof(T)); unmap(Data, CapacityBytes); diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.cpp b/compiler-rt/lib/scudo/standalone/wrappers_c.cpp index dffd7cc26fe83..93a666c4d61e3 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.cpp @@ -18,22 +18,23 @@ #include <stdint.h> #include <stdio.h> -static scudo::Allocator<scudo::Config> Allocator; +#define SCUDO_PREFIX(name) name +#define SCUDO_ALLOCATOR Allocator + +extern "C" void SCUDO_PREFIX(malloc_postinit)(); +static scudo::Allocator<scudo::Config, SCUDO_PREFIX(malloc_postinit)> + SCUDO_ALLOCATOR; // Pointer to the static allocator so that the C++ wrappers can access it. // Technically we could have a completely separated heap for C & C++ but in // reality the amount of cross pollination between the two is staggering. -scudo::Allocator<scudo::Config> *AllocatorPtr = &Allocator; - -extern "C" { +scudo::Allocator<scudo::Config, SCUDO_PREFIX(malloc_postinit)> * + CONCATENATE(SCUDO_ALLOCATOR, Ptr) = &SCUDO_ALLOCATOR; -#define SCUDO_PREFIX(name) name -#define SCUDO_ALLOCATOR Allocator #include "wrappers_c.inc" + #undef SCUDO_ALLOCATOR #undef SCUDO_PREFIX -INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); } - -} // extern "C" +extern "C" INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); } #endif // !SCUDO_ANDROID || !_BIONIC diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc index a9adbc83588ba..2fd709eaa1f62 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc @@ -17,6 +17,8 @@ #define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U) #endif +extern "C" { + INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) { scudo::uptr Product; if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) { @@ -134,26 +136,23 @@ INTERFACE WEAK void *SCUDO_PREFIX(valloc)(size_t size) { size, scudo::Chunk::Origin::Memalign, scudo::getPageSizeCached())); } -// Bionic wants a function named PREFIX_iterate and not PREFIX_malloc_iterate -// which is somewhat inconsistent with the rest, workaround that. -#if SCUDO_ANDROID && _BIONIC -#define SCUDO_ITERATE iterate -#else -#define SCUDO_ITERATE malloc_iterate -#endif - -INTERFACE WEAK int SCUDO_PREFIX(SCUDO_ITERATE)( +INTERFACE WEAK int SCUDO_PREFIX(malloc_iterate)( uintptr_t base, size_t size, void (*callback)(uintptr_t base, size_t size, void *arg), void *arg) { SCUDO_ALLOCATOR.iterateOverChunks(base, size, callback, arg); return 0; } +INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { SCUDO_ALLOCATOR.enable(); } + INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() { SCUDO_ALLOCATOR.disable(); } -INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { SCUDO_ALLOCATOR.enable(); } +void SCUDO_PREFIX(malloc_postinit)() { + pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable), + SCUDO_PREFIX(malloc_enable)); +} INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) { if (param == M_DECAY_TIME) { @@ -184,3 +183,5 @@ INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(UNUSED int options, FILE *stream) { fputs("</malloc>", stream); return 0; } + +} // extern "C" diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp index fa4145c066b64..f004369d96cbc 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp @@ -18,22 +18,40 @@ #include <stdint.h> #include <stdio.h> -static scudo::Allocator<scudo::AndroidConfig> Allocator; -static scudo::Allocator<scudo::AndroidSvelteConfig> SvelteAllocator; - -extern "C" { - // Regular MallocDispatch definitions. #define SCUDO_PREFIX(name) CONCATENATE(scudo_, name) #define SCUDO_ALLOCATOR Allocator + +extern "C" void SCUDO_PREFIX(malloc_postinit)(); +static scudo::Allocator<scudo::AndroidConfig, SCUDO_PREFIX(malloc_postinit)> + SCUDO_ALLOCATOR; +// Pointer to the static allocator so that the C++ wrappers can access it. +// Technically we could have a completely separated heap for C & C++ but in +// reality the amount of cross pollination between the two is staggering. +scudo::Allocator<scudo::AndroidConfig, SCUDO_PREFIX(malloc_postinit)> * + CONCATENATE(SCUDO_ALLOCATOR, Ptr) = &SCUDO_ALLOCATOR; + #include "wrappers_c.inc" + #undef SCUDO_ALLOCATOR #undef SCUDO_PREFIX // Svelte MallocDispatch definitions. #define SCUDO_PREFIX(name) CONCATENATE(scudo_svelte_, name) #define SCUDO_ALLOCATOR SvelteAllocator + +extern "C" void SCUDO_PREFIX(malloc_postinit)(); +static scudo::Allocator<scudo::AndroidSvelteConfig, + SCUDO_PREFIX(malloc_postinit)> + SCUDO_ALLOCATOR; +// Pointer to the static allocator so that the C++ wrappers can access it. +// Technically we could have a completely separated heap for C & C++ but in +// reality the amount of cross pollination between the two is staggering. +scudo::Allocator<scudo::AndroidSvelteConfig, SCUDO_PREFIX(malloc_postinit)> * + CONCATENATE(SCUDO_ALLOCATOR, Ptr) = &SCUDO_ALLOCATOR; + #include "wrappers_c.inc" + #undef SCUDO_ALLOCATOR #undef SCUDO_PREFIX @@ -44,6 +62,4 @@ INTERFACE void __scudo_print_stats(void) { SvelteAllocator.printStats(); } -} // extern "C" - #endif // SCUDO_ANDROID && _BIONIC diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h b/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h index d4370d506e5ea..7fc1a9600e53b 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h +++ b/compiler-rt/lib/scudo/standalone/wrappers_c_checks.h @@ -20,7 +20,7 @@ namespace scudo { // A common errno setting logic shared by almost all Scudo C wrappers. -INLINE void *setErrnoOnNull(void *Ptr) { +inline void *setErrnoOnNull(void *Ptr) { if (UNLIKELY(!Ptr)) errno = ENOMEM; return Ptr; @@ -30,14 +30,14 @@ INLINE void *setErrnoOnNull(void *Ptr) { // Checks aligned_alloc() parameters, verifies that the alignment is a power of // two and that the size is a multiple of alignment. -INLINE bool checkAlignedAllocAlignmentAndSize(uptr Alignment, uptr Size) { +inline bool checkAlignedAllocAlignmentAndSize(uptr Alignment, uptr Size) { return Alignment == 0 || !isPowerOfTwo(Alignment) || !isAligned(Size, Alignment); } // Checks posix_memalign() parameters, verifies that alignment is a power of two // and a multiple of sizeof(void *). -INLINE bool checkPosixMemalignAlignment(uptr Alignment) { +inline bool checkPosixMemalignAlignment(uptr Alignment) { return Alignment == 0 || !isPowerOfTwo(Alignment) || !isAligned(Alignment, sizeof(void *)); } @@ -45,7 +45,7 @@ INLINE bool checkPosixMemalignAlignment(uptr Alignment) { // Returns true if calloc(Size, N) overflows on Size*N calculation. Use a // builtin supported by recent clang & GCC if it exists, otherwise fallback to a // costly division. -INLINE bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) { +inline bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) { #if __has_builtin(__builtin_umull_overflow) return __builtin_umull_overflow(Size, N, Product); #else @@ -58,7 +58,7 @@ INLINE bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) { // Returns true if the size passed to pvalloc overflows when rounded to the next // multiple of PageSize. -INLINE bool checkForPvallocOverflow(uptr Size, uptr PageSize) { +inline bool checkForPvallocOverflow(uptr Size, uptr PageSize) { return roundUpTo(Size, PageSize) < Size; } diff --git a/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp b/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp index 72235e9c98209..1da5385c77890 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp @@ -15,7 +15,8 @@ #include <stdint.h> -extern scudo::Allocator<scudo::Config> *AllocatorPtr; +extern "C" void malloc_postinit(); +extern scudo::Allocator<scudo::Config, malloc_postinit> *AllocatorPtr; namespace std { struct nothrow_t {}; |