summaryrefslogtreecommitdiff
path: root/lib/scudo/scudo_allocator.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/scudo/scudo_allocator.cpp')
-rw-r--r--lib/scudo/scudo_allocator.cpp495
1 files changed, 262 insertions, 233 deletions
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index e5a4d714c66e..4a11bf5fcc21 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -16,7 +16,9 @@
#include "scudo_allocator.h"
#include "scudo_crc32.h"
+#include "scudo_errors.h"
#include "scudo_flags.h"
+#include "scudo_interface_internal.h"
#include "scudo_tsd.h"
#include "scudo_utils.h"
@@ -60,40 +62,49 @@ INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
}
-static ScudoBackendAllocator &getBackendAllocator();
+static BackendT &getBackend();
namespace Chunk {
- // We can't use the offset member of the chunk itself, as we would double
- // fetch it without any warranty that it wouldn't have been tampered. To
- // prevent this, we work with a local copy of the header.
- static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) {
- return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
- AlignedChunkHeaderSize -
- (Header->Offset << MinAlignmentLog));
- }
-
static INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) {
return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) -
- AlignedChunkHeaderSize);
+ getHeaderSize());
}
static INLINE
const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) {
return reinterpret_cast<const AtomicPackedHeader *>(
- reinterpret_cast<uptr>(Ptr) - AlignedChunkHeaderSize);
+ reinterpret_cast<uptr>(Ptr) - getHeaderSize());
}
static INLINE bool isAligned(const void *Ptr) {
return IsAligned(reinterpret_cast<uptr>(Ptr), MinAlignment);
}
+ // We can't use the offset member of the chunk itself, as we would double
+ // fetch it without any warranty that it wouldn't have been tampered. To
+ // prevent this, we work with a local copy of the header.
+ static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) {
+ return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
+ getHeaderSize() - (Header->Offset << MinAlignmentLog));
+ }
+
// Returns the usable size for a chunk, meaning the amount of bytes from the
// beginning of the user data to the end of the backend allocated chunk.
static INLINE uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) {
- const uptr Size = getBackendAllocator().getActuallyAllocatedSize(
- getBackendPtr(Ptr, Header), Header->ClassId);
- if (Size == 0)
- return 0;
- return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog);
+ const uptr ClassId = Header->ClassId;
+ if (ClassId)
+ return PrimaryT::ClassIdToSize(ClassId) - getHeaderSize() -
+ (Header->Offset << MinAlignmentLog);
+ return SecondaryT::GetActuallyAllocatedSize(
+ getBackendPtr(Ptr, Header)) - getHeaderSize();
+ }
+
+ // Returns the size the user requested when allocating the chunk.
+ static INLINE uptr getSize(const void *Ptr, UnpackedHeader *Header) {
+ const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
+ if (Header->ClassId)
+ return SizeOrUnusedBytes;
+ return SecondaryT::GetActuallyAllocatedSize(
+ getBackendPtr(Ptr, Header)) - getHeaderSize() - SizeOrUnusedBytes;
}
// Compute the checksum of the chunk pointer and its header.
@@ -136,9 +147,8 @@ namespace Chunk {
atomic_load_relaxed(getConstAtomicHeader(Ptr));
*NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
if (UNLIKELY(NewUnpackedHeader->Checksum !=
- computeChecksum(Ptr, NewUnpackedHeader))) {
- dieWithMessage("ERROR: corrupted chunk header at address %p\n", Ptr);
- }
+ computeChecksum(Ptr, NewUnpackedHeader)))
+ dieWithMessage("corrupted chunk header at address %p\n", Ptr);
}
// Packs and stores the header, computing the checksum in the process.
@@ -159,14 +169,13 @@ namespace Chunk {
PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader);
if (UNLIKELY(!atomic_compare_exchange_strong(
getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader,
- memory_order_relaxed))) {
- dieWithMessage("ERROR: race on chunk header at address %p\n", Ptr);
- }
+ memory_order_relaxed)))
+ dieWithMessage("race on chunk header at address %p\n", Ptr);
}
} // namespace Chunk
struct QuarantineCallback {
- explicit QuarantineCallback(AllocatorCache *Cache)
+ explicit QuarantineCallback(AllocatorCacheT *Cache)
: Cache_(Cache) {}
// Chunk recycling function, returns a quarantined chunk to the backend,
@@ -174,53 +183,48 @@ struct QuarantineCallback {
void Recycle(void *Ptr) {
UnpackedHeader Header;
Chunk::loadHeader(Ptr, &Header);
- if (UNLIKELY(Header.State != ChunkQuarantine)) {
- dieWithMessage("ERROR: invalid chunk state when recycling address %p\n",
- Ptr);
- }
+ if (UNLIKELY(Header.State != ChunkQuarantine))
+ dieWithMessage("invalid chunk state when recycling address %p\n", Ptr);
Chunk::eraseHeader(Ptr);
void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header);
if (Header.ClassId)
- getBackendAllocator().deallocatePrimary(Cache_, BackendPtr,
- Header.ClassId);
+ getBackend().deallocatePrimary(Cache_, BackendPtr, Header.ClassId);
else
- getBackendAllocator().deallocateSecondary(BackendPtr);
+ getBackend().deallocateSecondary(BackendPtr);
}
// Internal quarantine allocation and deallocation functions. We first check
// that the batches are indeed serviced by the Primary.
// TODO(kostyak): figure out the best way to protect the batches.
void *Allocate(uptr Size) {
- return getBackendAllocator().allocatePrimary(Cache_, BatchClassId);
+ const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
+ return getBackend().allocatePrimary(Cache_, BatchClassId);
}
void Deallocate(void *Ptr) {
- getBackendAllocator().deallocatePrimary(Cache_, Ptr, BatchClassId);
+ const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
+ getBackend().deallocatePrimary(Cache_, Ptr, BatchClassId);
}
- AllocatorCache *Cache_;
+ AllocatorCacheT *Cache_;
COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize);
- const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
};
-typedef Quarantine<QuarantineCallback, void> ScudoQuarantine;
-typedef ScudoQuarantine::Cache ScudoQuarantineCache;
-COMPILER_CHECK(sizeof(ScudoQuarantineCache) <=
+typedef Quarantine<QuarantineCallback, void> QuarantineT;
+typedef QuarantineT::Cache QuarantineCacheT;
+COMPILER_CHECK(sizeof(QuarantineCacheT) <=
sizeof(ScudoTSD::QuarantineCachePlaceHolder));
-ScudoQuarantineCache *getQuarantineCache(ScudoTSD *TSD) {
- return reinterpret_cast<ScudoQuarantineCache *>(
- TSD->QuarantineCachePlaceHolder);
+QuarantineCacheT *getQuarantineCache(ScudoTSD *TSD) {
+ return reinterpret_cast<QuarantineCacheT *>(TSD->QuarantineCachePlaceHolder);
}
-struct ScudoAllocator {
+struct Allocator {
static const uptr MaxAllowedMallocSize =
FIRST_32_SECOND_64(2UL << 30, 1ULL << 40);
- typedef ReturnNullOrDieOnFailure FailureHandler;
-
- ScudoBackendAllocator BackendAllocator;
- ScudoQuarantine AllocatorQuarantine;
+ BackendT Backend;
+ QuarantineT Quarantine;
u32 QuarantineChunksUpToSize;
@@ -234,49 +238,16 @@ struct ScudoAllocator {
atomic_uint8_t RssLimitExceeded;
atomic_uint64_t RssLastCheckedAtNS;
- explicit ScudoAllocator(LinkerInitialized)
- : AllocatorQuarantine(LINKER_INITIALIZED) {}
-
- void performSanityChecks() {
- // Verify that the header offset field can hold the maximum offset. In the
- // case of the Secondary allocator, it takes care of alignment and the
- // offset will always be 0. In the case of the Primary, the worst case
- // scenario happens in the last size class, when the backend allocation
- // would already be aligned on the requested alignment, which would happen
- // to be the maximum alignment that would fit in that size class. As a
- // result, the maximum offset will be at most the maximum alignment for the
- // last size class minus the header size, in multiples of MinAlignment.
- UnpackedHeader Header = {};
- const uptr MaxPrimaryAlignment =
- 1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment);
- const uptr MaxOffset =
- (MaxPrimaryAlignment - AlignedChunkHeaderSize) >> MinAlignmentLog;
- Header.Offset = MaxOffset;
- if (Header.Offset != MaxOffset) {
- dieWithMessage("ERROR: the maximum possible offset doesn't fit in the "
- "header\n");
- }
- // Verify that we can fit the maximum size or amount of unused bytes in the
- // header. Given that the Secondary fits the allocation to a page, the worst
- // case scenario happens in the Primary. It will depend on the second to
- // last and last class sizes, as well as the dynamic base for the Primary.
- // The following is an over-approximation that works for our needs.
- const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
- Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
- if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) {
- dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in "
- "the header\n");
- }
+ explicit Allocator(LinkerInitialized)
+ : Quarantine(LINKER_INITIALIZED) {}
- const uptr LargestClassId = SizeClassMap::kLargestClassID;
- Header.ClassId = LargestClassId;
- if (Header.ClassId != LargestClassId) {
- dieWithMessage("ERROR: the largest class ID doesn't fit in the header\n");
- }
- }
+ NOINLINE void performSanityChecks();
void init() {
SanitizerToolName = "Scudo";
+ PrimaryAllocatorName = "ScudoPrimary";
+ SecondaryAllocatorName = "ScudoSecondary";
+
initFlags();
performSanityChecks();
@@ -287,10 +258,10 @@ struct ScudoAllocator {
atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);
SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
- BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms);
+ Backend.init(common_flags()->allocator_release_to_os_interval_ms);
HardRssLimitMb = common_flags()->hard_rss_limit_mb;
SoftRssLimitMb = common_flags()->soft_rss_limit_mb;
- AllocatorQuarantine.Init(
+ Quarantine.Init(
static_cast<uptr>(getFlags()->QuarantineSizeKb) << 10,
static_cast<uptr>(getFlags()->ThreadLocalQuarantineSizeKb) << 10);
QuarantineChunksUpToSize = getFlags()->QuarantineChunksUpToSize;
@@ -319,62 +290,36 @@ struct ScudoAllocator {
return Chunk::isValid(Ptr);
}
- // Opportunistic RSS limit check. This will update the RSS limit status, if
- // it can, every 100ms, otherwise it will just return the current one.
- bool isRssLimitExceeded() {
- u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
- const u64 CurrentCheck = MonotonicNanoTime();
- if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
- return atomic_load_relaxed(&RssLimitExceeded);
- if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
- CurrentCheck, memory_order_relaxed))
- return atomic_load_relaxed(&RssLimitExceeded);
- // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the
- // RSS from /proc/self/statm by default. We might want to
- // call getrusage directly, even if it's less accurate.
- const uptr CurrentRssMb = GetRSS() >> 20;
- if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) {
- Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n",
- SanitizerToolName, HardRssLimitMb, CurrentRssMb);
- DumpProcessMap();
- Die();
- }
- if (SoftRssLimitMb) {
- if (atomic_load_relaxed(&RssLimitExceeded)) {
- if (CurrentRssMb <= SoftRssLimitMb)
- atomic_store_relaxed(&RssLimitExceeded, false);
- } else {
- if (CurrentRssMb > SoftRssLimitMb) {
- atomic_store_relaxed(&RssLimitExceeded, true);
- Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n",
- SanitizerToolName, SoftRssLimitMb, CurrentRssMb);
- }
- }
- }
- return atomic_load_relaxed(&RssLimitExceeded);
- }
+ NOINLINE bool isRssLimitExceeded();
// Allocates a chunk.
void *allocate(uptr Size, uptr Alignment, AllocType Type,
bool ForceZeroContents = false) {
initThreadMaybe();
- if (UNLIKELY(Alignment > MaxAlignment))
- return FailureHandler::OnBadRequest();
+ if (UNLIKELY(Alignment > MaxAlignment)) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ reportAllocationAlignmentTooBig(Alignment, MaxAlignment);
+ }
if (UNLIKELY(Alignment < MinAlignment))
Alignment = MinAlignment;
- if (UNLIKELY(Size >= MaxAllowedMallocSize))
- return FailureHandler::OnBadRequest();
- if (UNLIKELY(Size == 0))
- Size = 1;
- uptr NeededSize = RoundUpTo(Size, MinAlignment) + AlignedChunkHeaderSize;
- uptr AlignedSize = (Alignment > MinAlignment) ?
- NeededSize + (Alignment - AlignedChunkHeaderSize) : NeededSize;
- if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize))
- return FailureHandler::OnBadRequest();
+ const uptr NeededSize = RoundUpTo(Size ? Size : 1, MinAlignment) +
+ Chunk::getHeaderSize();
+ const uptr AlignedSize = (Alignment > MinAlignment) ?
+ NeededSize + (Alignment - Chunk::getHeaderSize()) : NeededSize;
+ if (UNLIKELY(Size >= MaxAllowedMallocSize) ||
+ UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ reportAllocationSizeTooBig(Size, AlignedSize, MaxAllowedMallocSize);
+ }
- if (CheckRssLimit && UNLIKELY(isRssLimitExceeded()))
- return FailureHandler::OnOOM();
+ if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ reportRssLimitExceeded();
+ }
// Primary and Secondary backed allocations have a different treatment. We
// deal with alignment requirements of Primary serviced allocations here,
@@ -382,27 +327,32 @@ struct ScudoAllocator {
void *BackendPtr;
uptr BackendSize;
u8 ClassId;
- if (PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment)) {
+ if (PrimaryT::CanAllocate(AlignedSize, MinAlignment)) {
BackendSize = AlignedSize;
ClassId = SizeClassMap::ClassID(BackendSize);
- ScudoTSD *TSD = getTSDAndLock();
- BackendPtr = BackendAllocator.allocatePrimary(&TSD->Cache, ClassId);
- TSD->unlock();
+ bool UnlockRequired;
+ ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
+ BackendPtr = Backend.allocatePrimary(&TSD->Cache, ClassId);
+ if (UnlockRequired)
+ TSD->unlock();
} else {
BackendSize = NeededSize;
ClassId = 0;
- BackendPtr = BackendAllocator.allocateSecondary(BackendSize, Alignment);
+ BackendPtr = Backend.allocateSecondary(BackendSize, Alignment);
+ }
+ if (UNLIKELY(!BackendPtr)) {
+ SetAllocatorOutOfMemory();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ reportOutOfMemory(Size);
}
- if (UNLIKELY(!BackendPtr))
- return FailureHandler::OnOOM();
// If requested, we will zero out the entire contents of the returned chunk.
if ((ForceZeroContents || ZeroContents) && ClassId)
- memset(BackendPtr, 0,
- BackendAllocator.getActuallyAllocatedSize(BackendPtr, ClassId));
+ memset(BackendPtr, 0, PrimaryT::ClassIdToSize(ClassId));
UnpackedHeader Header = {};
- uptr UserPtr = reinterpret_cast<uptr>(BackendPtr) + AlignedChunkHeaderSize;
+ uptr UserPtr = reinterpret_cast<uptr>(BackendPtr) + Chunk::getHeaderSize();
if (UNLIKELY(!IsAligned(UserPtr, Alignment))) {
// Since the Secondary takes care of alignment, a non-aligned pointer
// means it is from the Primary. It is also the only case where the offset
@@ -412,7 +362,7 @@ struct ScudoAllocator {
Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog;
UserPtr = AlignedUserPtr;
}
- CHECK_LE(UserPtr + Size, reinterpret_cast<uptr>(BackendPtr) + BackendSize);
+ DCHECK_LE(UserPtr + Size, reinterpret_cast<uptr>(BackendPtr) + BackendSize);
Header.State = ChunkAllocated;
Header.AllocType = Type;
if (ClassId) {
@@ -429,7 +379,8 @@ struct ScudoAllocator {
}
void *Ptr = reinterpret_cast<void *>(UserPtr);
Chunk::storeHeader(Ptr, &Header);
- // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(Ptr, Size);
+ if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook)
+ __sanitizer_malloc_hook(Ptr, Size);
return Ptr;
}
@@ -438,18 +389,20 @@ struct ScudoAllocator {
// quarantine chunk size threshold.
void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header,
uptr Size) {
- const bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0) ||
+ const bool BypassQuarantine = (Quarantine.GetCacheSize() == 0) ||
(Size > QuarantineChunksUpToSize);
if (BypassQuarantine) {
Chunk::eraseHeader(Ptr);
void *BackendPtr = Chunk::getBackendPtr(Ptr, Header);
if (Header->ClassId) {
- ScudoTSD *TSD = getTSDAndLock();
- getBackendAllocator().deallocatePrimary(&TSD->Cache, BackendPtr,
- Header->ClassId);
- TSD->unlock();
+ bool UnlockRequired;
+ ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
+ getBackend().deallocatePrimary(&TSD->Cache, BackendPtr,
+ Header->ClassId);
+ if (UnlockRequired)
+ TSD->unlock();
} else {
- getBackendAllocator().deallocateSecondary(BackendPtr);
+ getBackend().deallocateSecondary(BackendPtr);
}
} else {
// If a small memory amount was allocated with a larger alignment, we want
@@ -457,21 +410,23 @@ struct ScudoAllocator {
// with tiny chunks, taking a lot of VA memory. This is an approximation
// of the usable size, that allows us to not call
// GetActuallyAllocatedSize.
- uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog);
+ const uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog);
UnpackedHeader NewHeader = *Header;
NewHeader.State = ChunkQuarantine;
Chunk::compareExchangeHeader(Ptr, &NewHeader, Header);
- ScudoTSD *TSD = getTSDAndLock();
- AllocatorQuarantine.Put(getQuarantineCache(TSD),
- QuarantineCallback(&TSD->Cache), Ptr,
- EstimatedSize);
- TSD->unlock();
+ bool UnlockRequired;
+ ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
+ Quarantine.Put(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache),
+ Ptr, EstimatedSize);
+ if (UnlockRequired)
+ TSD->unlock();
}
}
// Deallocates a Chunk, which means either adding it to the quarantine or
// directly returning it to the backend if criteria are met.
- void deallocate(void *Ptr, uptr DeleteSize, AllocType Type) {
+ void deallocate(void *Ptr, uptr DeleteSize, uptr DeleteAlignment,
+ AllocType Type) {
// For a deallocation, we only ensure minimal initialization, meaning thread
// local data will be left uninitialized for now (when using ELF TLS). The
// fallback cache will be used instead. This is a workaround for a situation
@@ -479,37 +434,32 @@ struct ScudoAllocator {
// the TLS destructors, ending up in initialized thread specific data never
// being destroyed properly. Any other heap operation will do a full init.
initThreadMaybe(/*MinimalInit=*/true);
- // if (&__sanitizer_free_hook) __sanitizer_free_hook(Ptr);
+ if (SCUDO_CAN_USE_HOOKS && &__sanitizer_free_hook)
+ __sanitizer_free_hook(Ptr);
if (UNLIKELY(!Ptr))
return;
- if (UNLIKELY(!Chunk::isAligned(Ptr))) {
- dieWithMessage("ERROR: attempted to deallocate a chunk not properly "
- "aligned at address %p\n", Ptr);
- }
+ if (UNLIKELY(!Chunk::isAligned(Ptr)))
+ dieWithMessage("misaligned pointer when deallocating address %p\n", Ptr);
UnpackedHeader Header;
Chunk::loadHeader(Ptr, &Header);
- if (UNLIKELY(Header.State != ChunkAllocated)) {
- dieWithMessage("ERROR: invalid chunk state when deallocating address "
- "%p\n", Ptr);
- }
+ if (UNLIKELY(Header.State != ChunkAllocated))
+ dieWithMessage("invalid chunk state when deallocating address %p\n", Ptr);
if (DeallocationTypeMismatch) {
// The deallocation type has to match the allocation one.
if (Header.AllocType != Type) {
// With the exception of memalign'd Chunks, that can be still be free'd.
- if (Header.AllocType != FromMemalign || Type != FromMalloc) {
- dieWithMessage("ERROR: allocation type mismatch when deallocating "
- "address %p\n", Ptr);
- }
+ if (Header.AllocType != FromMemalign || Type != FromMalloc)
+ dieWithMessage("allocation type mismatch when deallocating address "
+ "%p\n", Ptr);
}
}
- uptr Size = Header.ClassId ? Header.SizeOrUnusedBytes :
- Chunk::getUsableSize(Ptr, &Header) - Header.SizeOrUnusedBytes;
+ const uptr Size = Chunk::getSize(Ptr, &Header);
if (DeleteSizeMismatch) {
- if (DeleteSize && DeleteSize != Size) {
- dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n",
+ if (DeleteSize && DeleteSize != Size)
+ dieWithMessage("invalid sized delete when deallocating address %p\n",
Ptr);
- }
}
+ (void)DeleteAlignment; // TODO(kostyak): verify that the alignment matches.
quarantineOrDeallocateChunk(Ptr, &Header, Size);
}
@@ -517,21 +467,18 @@ struct ScudoAllocator {
// size still fits in the chunk.
void *reallocate(void *OldPtr, uptr NewSize) {
initThreadMaybe();
- if (UNLIKELY(!Chunk::isAligned(OldPtr))) {
- dieWithMessage("ERROR: attempted to reallocate a chunk not properly "
- "aligned at address %p\n", OldPtr);
- }
+ if (UNLIKELY(!Chunk::isAligned(OldPtr)))
+ dieWithMessage("misaligned address when reallocating address %p\n",
+ OldPtr);
UnpackedHeader OldHeader;
Chunk::loadHeader(OldPtr, &OldHeader);
- if (UNLIKELY(OldHeader.State != ChunkAllocated)) {
- dieWithMessage("ERROR: invalid chunk state when reallocating address "
- "%p\n", OldPtr);
- }
+ if (UNLIKELY(OldHeader.State != ChunkAllocated))
+ dieWithMessage("invalid chunk state when reallocating address %p\n",
+ OldPtr);
if (DeallocationTypeMismatch) {
- if (UNLIKELY(OldHeader.AllocType != FromMalloc)) {
- dieWithMessage("ERROR: allocation type mismatch when reallocating "
- "address %p\n", OldPtr);
- }
+ if (UNLIKELY(OldHeader.AllocType != FromMalloc))
+ dieWithMessage("allocation type mismatch when reallocating address "
+ "%p\n", OldPtr);
}
const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader);
// The new size still fits in the current chunk, and the size difference
@@ -548,7 +495,7 @@ struct ScudoAllocator {
// old one.
void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
if (NewPtr) {
- uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes :
+ const uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes :
UsableSize - OldHeader.SizeOrUnusedBytes;
memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize));
quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize);
@@ -564,36 +511,36 @@ struct ScudoAllocator {
UnpackedHeader Header;
Chunk::loadHeader(Ptr, &Header);
// Getting the usable size of a chunk only makes sense if it's allocated.
- if (UNLIKELY(Header.State != ChunkAllocated)) {
- dieWithMessage("ERROR: invalid chunk state when sizing address %p\n",
- Ptr);
- }
+ if (UNLIKELY(Header.State != ChunkAllocated))
+ dieWithMessage("invalid chunk state when sizing address %p\n", Ptr);
return Chunk::getUsableSize(Ptr, &Header);
}
void *calloc(uptr NMemB, uptr Size) {
initThreadMaybe();
- if (UNLIKELY(CheckForCallocOverflow(NMemB, Size)))
- return FailureHandler::OnBadRequest();
+ if (UNLIKELY(CheckForCallocOverflow(NMemB, Size))) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ reportCallocOverflow(NMemB, Size);
+ }
return allocate(NMemB * Size, MinAlignment, FromMalloc, true);
}
void commitBack(ScudoTSD *TSD) {
- AllocatorQuarantine.Drain(getQuarantineCache(TSD),
- QuarantineCallback(&TSD->Cache));
- BackendAllocator.destroyCache(&TSD->Cache);
+ Quarantine.Drain(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache));
+ Backend.destroyCache(&TSD->Cache);
}
uptr getStats(AllocatorStat StatType) {
initThreadMaybe();
uptr stats[AllocatorStatCount];
- BackendAllocator.getStats(stats);
+ Backend.getStats(stats);
return stats[StatType];
}
- void *handleBadRequest() {
+ bool canReturnNull() {
initThreadMaybe();
- return FailureHandler::OnBadRequest();
+ return AllocatorMayReturnNull();
}
void setRssLimit(uptr LimitMb, bool HardLimit) {
@@ -603,21 +550,90 @@ struct ScudoAllocator {
SoftRssLimitMb = LimitMb;
CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
}
+
+ void printStats() {
+ initThreadMaybe();
+ Backend.printStats();
+ }
};
-static ScudoAllocator Instance(LINKER_INITIALIZED);
+NOINLINE void Allocator::performSanityChecks() {
+ // Verify that the header offset field can hold the maximum offset. In the
+ // case of the Secondary allocator, it takes care of alignment and the
+ // offset will always be 0. In the case of the Primary, the worst case
+ // scenario happens in the last size class, when the backend allocation
+ // would already be aligned on the requested alignment, which would happen
+ // to be the maximum alignment that would fit in that size class. As a
+ // result, the maximum offset will be at most the maximum alignment for the
+ // last size class minus the header size, in multiples of MinAlignment.
+ UnpackedHeader Header = {};
+ const uptr MaxPrimaryAlignment =
+ 1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment);
+ const uptr MaxOffset =
+ (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
+ Header.Offset = MaxOffset;
+ if (Header.Offset != MaxOffset)
+ dieWithMessage("maximum possible offset doesn't fit in header\n");
+ // Verify that we can fit the maximum size or amount of unused bytes in the
+ // header. Given that the Secondary fits the allocation to a page, the worst
+ // case scenario happens in the Primary. It will depend on the second to
+ // last and last class sizes, as well as the dynamic base for the Primary.
+ // The following is an over-approximation that works for our needs.
+ const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
+ Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
+ if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes)
+ dieWithMessage("maximum possible unused bytes doesn't fit in header\n");
+
+ const uptr LargestClassId = SizeClassMap::kLargestClassID;
+ Header.ClassId = LargestClassId;
+ if (Header.ClassId != LargestClassId)
+ dieWithMessage("largest class ID doesn't fit in header\n");
+}
-static ScudoBackendAllocator &getBackendAllocator() {
- return Instance.BackendAllocator;
+// Opportunistic RSS limit check. This will update the RSS limit status, if
+// it can, every 100ms, otherwise it will just return the current one.
+NOINLINE bool Allocator::isRssLimitExceeded() {
+ u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
+ const u64 CurrentCheck = MonotonicNanoTime();
+ if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
+ return atomic_load_relaxed(&RssLimitExceeded);
+ if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
+ CurrentCheck, memory_order_relaxed))
+ return atomic_load_relaxed(&RssLimitExceeded);
+ // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the
+ // RSS from /proc/self/statm by default. We might want to
+ // call getrusage directly, even if it's less accurate.
+ const uptr CurrentRssMb = GetRSS() >> 20;
+ if (HardRssLimitMb && UNLIKELY(HardRssLimitMb < CurrentRssMb))
+ dieWithMessage("hard RSS limit exhausted (%zdMb vs %zdMb)\n",
+ HardRssLimitMb, CurrentRssMb);
+ if (SoftRssLimitMb) {
+ if (atomic_load_relaxed(&RssLimitExceeded)) {
+ if (CurrentRssMb <= SoftRssLimitMb)
+ atomic_store_relaxed(&RssLimitExceeded, false);
+ } else {
+ if (CurrentRssMb > SoftRssLimitMb) {
+ atomic_store_relaxed(&RssLimitExceeded, true);
+ Printf("Scudo INFO: soft RSS limit exhausted (%zdMb vs %zdMb)\n",
+ SoftRssLimitMb, CurrentRssMb);
+ }
+ }
+ }
+ return atomic_load_relaxed(&RssLimitExceeded);
+}
+
+static Allocator Instance(LINKER_INITIALIZED);
+
+static BackendT &getBackend() {
+ return Instance.Backend;
}
void initScudo() {
Instance.init();
}
-void ScudoTSD::init(bool Shared) {
- UnlockRequired = Shared;
- getBackendAllocator().initCache(&Cache);
+void ScudoTSD::init() {
+ getBackend().initCache(&Cache);
memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
}
@@ -625,23 +641,25 @@ void ScudoTSD::commitBack() {
Instance.commitBack(this);
}
-void *scudoMalloc(uptr Size, AllocType Type) {
- return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, Type));
-}
-
-void scudoFree(void *Ptr, AllocType Type) {
- Instance.deallocate(Ptr, 0, Type);
+void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type) {
+ if (Alignment && UNLIKELY(!IsPowerOfTwo(Alignment))) {
+ errno = EINVAL;
+ if (Instance.canReturnNull())
+ return nullptr;
+ reportAllocationAlignmentNotPowerOfTwo(Alignment);
+ }
+ return SetErrnoOnNull(Instance.allocate(Size, Alignment, Type));
}
-void scudoSizedFree(void *Ptr, uptr Size, AllocType Type) {
- Instance.deallocate(Ptr, Size, Type);
+void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type) {
+ Instance.deallocate(Ptr, Size, Alignment, Type);
}
void *scudoRealloc(void *Ptr, uptr Size) {
if (!Ptr)
return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, FromMalloc));
if (Size == 0) {
- Instance.deallocate(Ptr, 0, FromMalloc);
+ Instance.deallocate(Ptr, 0, 0, FromMalloc);
return nullptr;
}
return SetErrnoOnNull(Instance.reallocate(Ptr, Size));
@@ -660,24 +678,19 @@ void *scudoPvalloc(uptr Size) {
uptr PageSize = GetPageSizeCached();
if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) {
errno = ENOMEM;
- return Instance.handleBadRequest();
+ if (Instance.canReturnNull())
+ return nullptr;
+ reportPvallocOverflow(Size);
}
// pvalloc(0) should allocate one page.
Size = Size ? RoundUpTo(Size, PageSize) : PageSize;
return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign));
}
-void *scudoMemalign(uptr Alignment, uptr Size) {
- if (UNLIKELY(!IsPowerOfTwo(Alignment))) {
- errno = EINVAL;
- return Instance.handleBadRequest();
- }
- return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMemalign));
-}
-
int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) {
if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) {
- Instance.handleBadRequest();
+ if (!Instance.canReturnNull())
+ reportInvalidPosixMemalignAlignment(Alignment);
return EINVAL;
}
void *Ptr = Instance.allocate(Size, Alignment, FromMemalign);
@@ -690,7 +703,9 @@ int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) {
void *scudoAlignedAlloc(uptr Alignment, uptr Size) {
if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) {
errno = EINVAL;
- return Instance.handleBadRequest();
+ if (Instance.canReturnNull())
+ return nullptr;
+ reportInvalidAlignedAllocAlignment(Size, Alignment);
}
return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc));
}
@@ -721,8 +736,8 @@ uptr __sanitizer_get_unmapped_bytes() {
return 1;
}
-uptr __sanitizer_get_estimated_allocated_size(uptr size) {
- return size;
+uptr __sanitizer_get_estimated_allocated_size(uptr Size) {
+ return Size;
}
int __sanitizer_get_ownership(const void *Ptr) {
@@ -733,12 +748,26 @@ uptr __sanitizer_get_allocated_size(const void *Ptr) {
return Instance.getUsableSize(Ptr);
}
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook,
+ void *Ptr, uptr Size) {
+ (void)Ptr;
+ (void)Size;
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *Ptr) {
+ (void)Ptr;
+}
+#endif
+
// Interface functions
-extern "C" {
-void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit) { // NOLINT
+void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit) {
if (!SCUDO_CAN_USE_PUBLIC_INTERFACE)
return;
Instance.setRssLimit(LimitMb, !!HardLimit);
}
-} // extern "C"
+
+void __scudo_print_stats() {
+ Instance.printStats();
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-10 06:24:39 +0000