vendor/compiler-rt/compiler-rt-trunk-r338150

1 files changed, 262 insertions, 233 deletions

diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index e5a4d714c66e..4a11bf5fcc21 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -16,7 +16,9 @@
 
 #include "scudo_allocator.h"
 #include "scudo_crc32.h"
+#include "scudo_errors.h"
 #include "scudo_flags.h"
+#include "scudo_interface_internal.h"
 #include "scudo_tsd.h"
 #include "scudo_utils.h"
 
@@ -60,40 +62,49 @@ INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
 #endif  // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
 }
 
-static ScudoBackendAllocator &getBackendAllocator();
+static BackendT &getBackend();
 
 namespace Chunk {
-  // We can't use the offset member of the chunk itself, as we would double
-  // fetch it without any warranty that it wouldn't have been tampered. To
-  // prevent this, we work with a local copy of the header.
-  static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) {
-    return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
-                                    AlignedChunkHeaderSize -
-                                    (Header->Offset << MinAlignmentLog));
-  }
-
   static INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) {
     return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) -
-                                                  AlignedChunkHeaderSize);
+        getHeaderSize());
   }
   static INLINE
   const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) {
     return reinterpret_cast<const AtomicPackedHeader *>(
-        reinterpret_cast<uptr>(Ptr) - AlignedChunkHeaderSize);
+        reinterpret_cast<uptr>(Ptr) - getHeaderSize());
   }
 
   static INLINE bool isAligned(const void *Ptr) {
     return IsAligned(reinterpret_cast<uptr>(Ptr), MinAlignment);
   }
 
+  // We can't use the offset member of the chunk itself, as we would double
+  // fetch it without any warranty that it wouldn't have been tampered. To
+  // prevent this, we work with a local copy of the header.
+  static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) {
+    return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
+        getHeaderSize() - (Header->Offset << MinAlignmentLog));
+  }
+
   // Returns the usable size for a chunk, meaning the amount of bytes from the
   // beginning of the user data to the end of the backend allocated chunk.
   static INLINE uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) {
-    const uptr Size = getBackendAllocator().getActuallyAllocatedSize(
-        getBackendPtr(Ptr, Header), Header->ClassId);
-    if (Size == 0)
-      return 0;
-    return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog);
+    const uptr ClassId = Header->ClassId;
+    if (ClassId)
+      return PrimaryT::ClassIdToSize(ClassId) - getHeaderSize() -
+          (Header->Offset << MinAlignmentLog);
+    return SecondaryT::GetActuallyAllocatedSize(
+        getBackendPtr(Ptr, Header)) - getHeaderSize();
+  }
+
+  // Returns the size the user requested when allocating the chunk.
+  static INLINE uptr getSize(const void *Ptr, UnpackedHeader *Header) {
+    const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
+    if (Header->ClassId)
+      return SizeOrUnusedBytes;
+    return SecondaryT::GetActuallyAllocatedSize(
+        getBackendPtr(Ptr, Header)) - getHeaderSize() - SizeOrUnusedBytes;
   }
 
   // Compute the checksum of the chunk pointer and its header.
@@ -136,9 +147,8 @@ namespace Chunk {
         atomic_load_relaxed(getConstAtomicHeader(Ptr));
     *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
     if (UNLIKELY(NewUnpackedHeader->Checksum !=
-        computeChecksum(Ptr, NewUnpackedHeader))) {
-      dieWithMessage("ERROR: corrupted chunk header at address %p\n", Ptr);
-    }
+        computeChecksum(Ptr, NewUnpackedHeader)))
+      dieWithMessage("corrupted chunk header at address %p\n", Ptr);
   }
 
   // Packs and stores the header, computing the checksum in the process.
@@ -159,14 +169,13 @@ namespace Chunk {
     PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader);
     if (UNLIKELY(!atomic_compare_exchange_strong(
             getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader,
-            memory_order_relaxed))) {
-      dieWithMessage("ERROR: race on chunk header at address %p\n", Ptr);
-    }
+            memory_order_relaxed)))
+      dieWithMessage("race on chunk header at address %p\n", Ptr);
   }
 }  // namespace Chunk
 
 struct QuarantineCallback {
-  explicit QuarantineCallback(AllocatorCache *Cache)
+  explicit QuarantineCallback(AllocatorCacheT *Cache)
     : Cache_(Cache) {}
 
   // Chunk recycling function, returns a quarantined chunk to the backend,
@@ -174,53 +183,48 @@ struct QuarantineCallback {
   void Recycle(void *Ptr) {
     UnpackedHeader Header;
     Chunk::loadHeader(Ptr, &Header);
-    if (UNLIKELY(Header.State != ChunkQuarantine)) {
-      dieWithMessage("ERROR: invalid chunk state when recycling address %p\n",
-                     Ptr);
-    }
+    if (UNLIKELY(Header.State != ChunkQuarantine))
+      dieWithMessage("invalid chunk state when recycling address %p\n", Ptr);
     Chunk::eraseHeader(Ptr);
     void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header);
     if (Header.ClassId)
-      getBackendAllocator().deallocatePrimary(Cache_, BackendPtr,
-                                              Header.ClassId);
+      getBackend().deallocatePrimary(Cache_, BackendPtr, Header.ClassId);
     else
-      getBackendAllocator().deallocateSecondary(BackendPtr);
+      getBackend().deallocateSecondary(BackendPtr);
   }
 
   // Internal quarantine allocation and deallocation functions. We first check
   // that the batches are indeed serviced by the Primary.
   // TODO(kostyak): figure out the best way to protect the batches.
   void *Allocate(uptr Size) {
-    return getBackendAllocator().allocatePrimary(Cache_, BatchClassId);
+    const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
+    return getBackend().allocatePrimary(Cache_, BatchClassId);
   }
 
   void Deallocate(void *Ptr) {
-    getBackendAllocator().deallocatePrimary(Cache_, Ptr, BatchClassId);
+    const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
+    getBackend().deallocatePrimary(Cache_, Ptr, BatchClassId);
   }
 
-  AllocatorCache *Cache_;
+  AllocatorCacheT *Cache_;
   COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize);
-  const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
 };
 
-typedef Quarantine<QuarantineCallback, void> ScudoQuarantine;
-typedef ScudoQuarantine::Cache ScudoQuarantineCache;
-COMPILER_CHECK(sizeof(ScudoQuarantineCache) <=
+typedef Quarantine<QuarantineCallback, void> QuarantineT;
+typedef QuarantineT::Cache QuarantineCacheT;
+COMPILER_CHECK(sizeof(QuarantineCacheT) <=
                sizeof(ScudoTSD::QuarantineCachePlaceHolder));
 
-ScudoQuarantineCache *getQuarantineCache(ScudoTSD *TSD) {
-  return reinterpret_cast<ScudoQuarantineCache *>(
-      TSD->QuarantineCachePlaceHolder);
+QuarantineCacheT *getQuarantineCache(ScudoTSD *TSD) {
+  return reinterpret_cast<QuarantineCacheT *>(TSD->QuarantineCachePlaceHolder);
 }
 
-struct ScudoAllocator {
+struct Allocator {
   static const uptr MaxAllowedMallocSize =
       FIRST_32_SECOND_64(2UL << 30, 1ULL << 40);
 
-  typedef ReturnNullOrDieOnFailure FailureHandler;
-
-  ScudoBackendAllocator BackendAllocator;
-  ScudoQuarantine AllocatorQuarantine;
+  BackendT Backend;
+  QuarantineT Quarantine;
 
   u32 QuarantineChunksUpToSize;
 
@@ -234,49 +238,16 @@ struct ScudoAllocator {
   atomic_uint8_t RssLimitExceeded;
   atomic_uint64_t RssLastCheckedAtNS;
 
-  explicit ScudoAllocator(LinkerInitialized)
-    : AllocatorQuarantine(LINKER_INITIALIZED) {}
-
-  void performSanityChecks() {
-    // Verify that the header offset field can hold the maximum offset. In the
-    // case of the Secondary allocator, it takes care of alignment and the
-    // offset will always be 0. In the case of the Primary, the worst case
-    // scenario happens in the last size class, when the backend allocation
-    // would already be aligned on the requested alignment, which would happen
-    // to be the maximum alignment that would fit in that size class. As a
-    // result, the maximum offset will be at most the maximum alignment for the
-    // last size class minus the header size, in multiples of MinAlignment.
-    UnpackedHeader Header = {};
-    const uptr MaxPrimaryAlignment =
-        1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment);
-    const uptr MaxOffset =
-        (MaxPrimaryAlignment - AlignedChunkHeaderSize) >> MinAlignmentLog;
-    Header.Offset = MaxOffset;
-    if (Header.Offset != MaxOffset) {
-      dieWithMessage("ERROR: the maximum possible offset doesn't fit in the "
-                     "header\n");
-    }
-    // Verify that we can fit the maximum size or amount of unused bytes in the
-    // header. Given that the Secondary fits the allocation to a page, the worst
-    // case scenario happens in the Primary. It will depend on the second to
-    // last and last class sizes, as well as the dynamic base for the Primary.
-    // The following is an over-approximation that works for our needs.
-    const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
-    Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
-    if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) {
-      dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in "
-                     "the header\n");
-    }
+  explicit Allocator(LinkerInitialized)
+    : Quarantine(LINKER_INITIALIZED) {}
 
-    const uptr LargestClassId = SizeClassMap::kLargestClassID;
-    Header.ClassId = LargestClassId;
-    if (Header.ClassId != LargestClassId) {
-      dieWithMessage("ERROR: the largest class ID doesn't fit in the header\n");
-    }
-  }
+  NOINLINE void performSanityChecks();
 
   void init() {
     SanitizerToolName = "Scudo";
+    PrimaryAllocatorName = "ScudoPrimary";
+    SecondaryAllocatorName = "ScudoSecondary";
+
     initFlags();
 
     performSanityChecks();
@@ -287,10 +258,10 @@ struct ScudoAllocator {
       atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);
 
     SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
-    BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms);
+    Backend.init(common_flags()->allocator_release_to_os_interval_ms);
     HardRssLimitMb = common_flags()->hard_rss_limit_mb;
     SoftRssLimitMb = common_flags()->soft_rss_limit_mb;
-    AllocatorQuarantine.Init(
+    Quarantine.Init(
         static_cast<uptr>(getFlags()->QuarantineSizeKb) << 10,
         static_cast<uptr>(getFlags()->ThreadLocalQuarantineSizeKb) << 10);
     QuarantineChunksUpToSize = getFlags()->QuarantineChunksUpToSize;
@@ -319,62 +290,36 @@ struct ScudoAllocator {
     return Chunk::isValid(Ptr);
   }
 
-  // Opportunistic RSS limit check. This will update the RSS limit status, if
-  // it can, every 100ms, otherwise it will just return the current one.
-  bool isRssLimitExceeded() {
-    u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
-    const u64 CurrentCheck = MonotonicNanoTime();
-    if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
-      return atomic_load_relaxed(&RssLimitExceeded);
-    if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
-                                      CurrentCheck, memory_order_relaxed))
-      return atomic_load_relaxed(&RssLimitExceeded);
-    // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the
-    //                RSS from /proc/self/statm by default. We might want to
-    //                call getrusage directly, even if it's less accurate.
-    const uptr CurrentRssMb = GetRSS() >> 20;
-    if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) {
-      Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n",
-             SanitizerToolName, HardRssLimitMb, CurrentRssMb);
-      DumpProcessMap();
-      Die();
-    }
-    if (SoftRssLimitMb) {
-      if (atomic_load_relaxed(&RssLimitExceeded)) {
-        if (CurrentRssMb <= SoftRssLimitMb)
-          atomic_store_relaxed(&RssLimitExceeded, false);
-      } else {
-        if (CurrentRssMb > SoftRssLimitMb) {
-          atomic_store_relaxed(&RssLimitExceeded, true);
-          Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n",
-                 SanitizerToolName, SoftRssLimitMb, CurrentRssMb);
-        }
-      }
-    }
-    return atomic_load_relaxed(&RssLimitExceeded);
-  }
+  NOINLINE bool isRssLimitExceeded();
 
   // Allocates a chunk.
   void *allocate(uptr Size, uptr Alignment, AllocType Type,
                  bool ForceZeroContents = false) {
     initThreadMaybe();
-    if (UNLIKELY(Alignment > MaxAlignment))
-      return FailureHandler::OnBadRequest();
+    if (UNLIKELY(Alignment > MaxAlignment)) {
+      if (AllocatorMayReturnNull())
+        return nullptr;
+      reportAllocationAlignmentTooBig(Alignment, MaxAlignment);
+    }
     if (UNLIKELY(Alignment < MinAlignment))
       Alignment = MinAlignment;
-    if (UNLIKELY(Size >= MaxAllowedMallocSize))
-      return FailureHandler::OnBadRequest();
-    if (UNLIKELY(Size == 0))
-      Size = 1;
 
-    uptr NeededSize = RoundUpTo(Size, MinAlignment) + AlignedChunkHeaderSize;
-    uptr AlignedSize = (Alignment > MinAlignment) ?
-        NeededSize + (Alignment - AlignedChunkHeaderSize) : NeededSize;
-    if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize))
-      return FailureHandler::OnBadRequest();
+    const uptr NeededSize = RoundUpTo(Size ? Size : 1, MinAlignment) +
+        Chunk::getHeaderSize();
+    const uptr AlignedSize = (Alignment > MinAlignment) ?
+        NeededSize + (Alignment - Chunk::getHeaderSize()) : NeededSize;
+    if (UNLIKELY(Size >= MaxAllowedMallocSize) ||
+        UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) {
+      if (AllocatorMayReturnNull())
+        return nullptr;
+      reportAllocationSizeTooBig(Size, AlignedSize, MaxAllowedMallocSize);
+    }
 
-    if (CheckRssLimit && UNLIKELY(isRssLimitExceeded()))
-      return FailureHandler::OnOOM();
+    if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) {
+      if (AllocatorMayReturnNull())
+        return nullptr;
+      reportRssLimitExceeded();
+    }
 
     // Primary and Secondary backed allocations have a different treatment. We
     // deal with alignment requirements of Primary serviced allocations here,
@@ -382,27 +327,32 @@ struct ScudoAllocator {
     void *BackendPtr;
     uptr BackendSize;
     u8 ClassId;
-    if (PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment)) {
+    if (PrimaryT::CanAllocate(AlignedSize, MinAlignment)) {
       BackendSize = AlignedSize;
       ClassId = SizeClassMap::ClassID(BackendSize);
-      ScudoTSD *TSD = getTSDAndLock();
-      BackendPtr = BackendAllocator.allocatePrimary(&TSD->Cache, ClassId);
-      TSD->unlock();
+      bool UnlockRequired;
+      ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
+      BackendPtr = Backend.allocatePrimary(&TSD->Cache, ClassId);
+      if (UnlockRequired)
+        TSD->unlock();
     } else {
       BackendSize = NeededSize;
       ClassId = 0;
-      BackendPtr = BackendAllocator.allocateSecondary(BackendSize, Alignment);
+      BackendPtr = Backend.allocateSecondary(BackendSize, Alignment);
+    }
+    if (UNLIKELY(!BackendPtr)) {
+      SetAllocatorOutOfMemory();
+      if (AllocatorMayReturnNull())
+        return nullptr;
+      reportOutOfMemory(Size);
     }
-    if (UNLIKELY(!BackendPtr))
-      return FailureHandler::OnOOM();
 
     // If requested, we will zero out the entire contents of the returned chunk.
     if ((ForceZeroContents || ZeroContents) && ClassId)
-      memset(BackendPtr, 0,
-             BackendAllocator.getActuallyAllocatedSize(BackendPtr, ClassId));
+      memset(BackendPtr, 0, PrimaryT::ClassIdToSize(ClassId));
 
     UnpackedHeader Header = {};
-    uptr UserPtr = reinterpret_cast<uptr>(BackendPtr) + AlignedChunkHeaderSize;
+    uptr UserPtr = reinterpret_cast<uptr>(BackendPtr) + Chunk::getHeaderSize();
     if (UNLIKELY(!IsAligned(UserPtr, Alignment))) {
       // Since the Secondary takes care of alignment, a non-aligned pointer
       // means it is from the Primary. It is also the only case where the offset
@@ -412,7 +362,7 @@ struct ScudoAllocator {
       Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog;
       UserPtr = AlignedUserPtr;
     }
-    CHECK_LE(UserPtr + Size, reinterpret_cast<uptr>(BackendPtr) + BackendSize);
+    DCHECK_LE(UserPtr + Size, reinterpret_cast<uptr>(BackendPtr) + BackendSize);
     Header.State = ChunkAllocated;
     Header.AllocType = Type;
     if (ClassId) {
@@ -429,7 +379,8 @@ struct ScudoAllocator {
     }
     void *Ptr = reinterpret_cast<void *>(UserPtr);
     Chunk::storeHeader(Ptr, &Header);
-    // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(Ptr, Size);
+    if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook)
+      __sanitizer_malloc_hook(Ptr, Size);
     return Ptr;
   }
 
@@ -438,18 +389,20 @@ struct ScudoAllocator {
   // quarantine chunk size threshold.
   void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header,
                                    uptr Size) {
-    const bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0) ||
+    const bool BypassQuarantine = (Quarantine.GetCacheSize() == 0) ||
         (Size > QuarantineChunksUpToSize);
     if (BypassQuarantine) {
       Chunk::eraseHeader(Ptr);
       void *BackendPtr = Chunk::getBackendPtr(Ptr, Header);
       if (Header->ClassId) {
-        ScudoTSD *TSD = getTSDAndLock();
-        getBackendAllocator().deallocatePrimary(&TSD->Cache, BackendPtr,
-                                                Header->ClassId);
-        TSD->unlock();
+        bool UnlockRequired;
+        ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
+        getBackend().deallocatePrimary(&TSD->Cache, BackendPtr,
+                                       Header->ClassId);
+        if (UnlockRequired)
+          TSD->unlock();
       } else {
-        getBackendAllocator().deallocateSecondary(BackendPtr);
+        getBackend().deallocateSecondary(BackendPtr);
       }
     } else {
       // If a small memory amount was allocated with a larger alignment, we want
@@ -457,21 +410,23 @@ struct ScudoAllocator {
       // with tiny chunks, taking a lot of VA memory. This is an approximation
       // of the usable size, that allows us to not call
       // GetActuallyAllocatedSize.
-      uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog);
+      const uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog);
       UnpackedHeader NewHeader = *Header;
       NewHeader.State = ChunkQuarantine;
       Chunk::compareExchangeHeader(Ptr, &NewHeader, Header);
-      ScudoTSD *TSD = getTSDAndLock();
-      AllocatorQuarantine.Put(getQuarantineCache(TSD),
-                              QuarantineCallback(&TSD->Cache), Ptr,
-                              EstimatedSize);
-      TSD->unlock();
+      bool UnlockRequired;
+      ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
+      Quarantine.Put(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache),
+                     Ptr, EstimatedSize);
+      if (UnlockRequired)
+        TSD->unlock();
     }
   }
 
   // Deallocates a Chunk, which means either adding it to the quarantine or
   // directly returning it to the backend if criteria are met.
-  void deallocate(void *Ptr, uptr DeleteSize, AllocType Type) {
+  void deallocate(void *Ptr, uptr DeleteSize, uptr DeleteAlignment,
+                  AllocType Type) {
     // For a deallocation, we only ensure minimal initialization, meaning thread
     // local data will be left uninitialized for now (when using ELF TLS). The
     // fallback cache will be used instead. This is a workaround for a situation
@@ -479,37 +434,32 @@ struct ScudoAllocator {
     // the TLS destructors, ending up in initialized thread specific data never
     // being destroyed properly. Any other heap operation will do a full init.
     initThreadMaybe(/*MinimalInit=*/true);
-    // if (&__sanitizer_free_hook) __sanitizer_free_hook(Ptr);
+    if (SCUDO_CAN_USE_HOOKS && &__sanitizer_free_hook)
+      __sanitizer_free_hook(Ptr);
     if (UNLIKELY(!Ptr))
       return;
-    if (UNLIKELY(!Chunk::isAligned(Ptr))) {
-      dieWithMessage("ERROR: attempted to deallocate a chunk not properly "
-                     "aligned at address %p\n", Ptr);
-    }
+    if (UNLIKELY(!Chunk::isAligned(Ptr)))
+      dieWithMessage("misaligned pointer when deallocating address %p\n", Ptr);
     UnpackedHeader Header;
     Chunk::loadHeader(Ptr, &Header);
-    if (UNLIKELY(Header.State != ChunkAllocated)) {
-      dieWithMessage("ERROR: invalid chunk state when deallocating address "
-                     "%p\n", Ptr);
-    }
+    if (UNLIKELY(Header.State != ChunkAllocated))
+      dieWithMessage("invalid chunk state when deallocating address %p\n", Ptr);
     if (DeallocationTypeMismatch) {
       // The deallocation type has to match the allocation one.
       if (Header.AllocType != Type) {
         // With the exception of memalign'd Chunks, that can be still be free'd.
-        if (Header.AllocType != FromMemalign || Type != FromMalloc) {
-          dieWithMessage("ERROR: allocation type mismatch when deallocating "
-                         "address %p\n", Ptr);
-        }
+        if (Header.AllocType != FromMemalign || Type != FromMalloc)
+          dieWithMessage("allocation type mismatch when deallocating address "
+                         "%p\n", Ptr);
       }
     }
-    uptr Size = Header.ClassId ? Header.SizeOrUnusedBytes :
-        Chunk::getUsableSize(Ptr, &Header) - Header.SizeOrUnusedBytes;
+    const uptr Size = Chunk::getSize(Ptr, &Header);
     if (DeleteSizeMismatch) {
-      if (DeleteSize && DeleteSize != Size) {
-        dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n",
+      if (DeleteSize && DeleteSize != Size)
+        dieWithMessage("invalid sized delete when deallocating address %p\n",
                        Ptr);
-      }
     }
+    (void)DeleteAlignment;  // TODO(kostyak): verify that the alignment matches.
     quarantineOrDeallocateChunk(Ptr, &Header, Size);
   }
 
@@ -517,21 +467,18 @@ struct ScudoAllocator {
   // size still fits in the chunk.
   void *reallocate(void *OldPtr, uptr NewSize) {
     initThreadMaybe();
-    if (UNLIKELY(!Chunk::isAligned(OldPtr))) {
-      dieWithMessage("ERROR: attempted to reallocate a chunk not properly "
-                     "aligned at address %p\n", OldPtr);
-    }
+    if (UNLIKELY(!Chunk::isAligned(OldPtr)))
+      dieWithMessage("misaligned address when reallocating address %p\n",
+                     OldPtr);
     UnpackedHeader OldHeader;
     Chunk::loadHeader(OldPtr, &OldHeader);
-    if (UNLIKELY(OldHeader.State != ChunkAllocated)) {
-      dieWithMessage("ERROR: invalid chunk state when reallocating address "
-                     "%p\n", OldPtr);
-    }
+    if (UNLIKELY(OldHeader.State != ChunkAllocated))
+      dieWithMessage("invalid chunk state when reallocating address %p\n",
+                     OldPtr);
     if (DeallocationTypeMismatch) {
-      if (UNLIKELY(OldHeader.AllocType != FromMalloc)) {
-        dieWithMessage("ERROR: allocation type mismatch when reallocating "
-                       "address %p\n", OldPtr);
-      }
+      if (UNLIKELY(OldHeader.AllocType != FromMalloc))
+        dieWithMessage("allocation type mismatch when reallocating address "
+                       "%p\n", OldPtr);
     }
     const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader);
     // The new size still fits in the current chunk, and the size difference
@@ -548,7 +495,7 @@ struct ScudoAllocator {
     // old one.
     void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
     if (NewPtr) {
-      uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes :
+      const uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes :
           UsableSize - OldHeader.SizeOrUnusedBytes;
       memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize));
       quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize);
@@ -564,36 +511,36 @@ struct ScudoAllocator {
     UnpackedHeader Header;
     Chunk::loadHeader(Ptr, &Header);
     // Getting the usable size of a chunk only makes sense if it's allocated.
-    if (UNLIKELY(Header.State != ChunkAllocated)) {
-      dieWithMessage("ERROR: invalid chunk state when sizing address %p\n",
-                     Ptr);
-    }
+    if (UNLIKELY(Header.State != ChunkAllocated))
+      dieWithMessage("invalid chunk state when sizing address %p\n", Ptr);
     return Chunk::getUsableSize(Ptr, &Header);
   }
 
   void *calloc(uptr NMemB, uptr Size) {
     initThreadMaybe();
-    if (UNLIKELY(CheckForCallocOverflow(NMemB, Size)))
-      return FailureHandler::OnBadRequest();
+    if (UNLIKELY(CheckForCallocOverflow(NMemB, Size))) {
+      if (AllocatorMayReturnNull())
+        return nullptr;
+      reportCallocOverflow(NMemB, Size);
+    }
     return allocate(NMemB * Size, MinAlignment, FromMalloc, true);
   }
 
   void commitBack(ScudoTSD *TSD) {
-    AllocatorQuarantine.Drain(getQuarantineCache(TSD),
-                              QuarantineCallback(&TSD->Cache));
-    BackendAllocator.destroyCache(&TSD->Cache);
+    Quarantine.Drain(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache));
+    Backend.destroyCache(&TSD->Cache);
   }
 
   uptr getStats(AllocatorStat StatType) {
     initThreadMaybe();
     uptr stats[AllocatorStatCount];
-    BackendAllocator.getStats(stats);
+    Backend.getStats(stats);
     return stats[StatType];
   }
 
-  void *handleBadRequest() {
+  bool canReturnNull() {
     initThreadMaybe();
-    return FailureHandler::OnBadRequest();
+    return AllocatorMayReturnNull();
   }
 
   void setRssLimit(uptr LimitMb, bool HardLimit) {
@@ -603,21 +550,90 @@ struct ScudoAllocator {
       SoftRssLimitMb = LimitMb;
     CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
   }
+
+  void printStats() {
+    initThreadMaybe();
+    Backend.printStats();
+  }
 };
 
-static ScudoAllocator Instance(LINKER_INITIALIZED);
+NOINLINE void Allocator::performSanityChecks() {
+  // Verify that the header offset field can hold the maximum offset. In the
+  // case of the Secondary allocator, it takes care of alignment and the
+  // offset will always be 0. In the case of the Primary, the worst case
+  // scenario happens in the last size class, when the backend allocation
+  // would already be aligned on the requested alignment, which would happen
+  // to be the maximum alignment that would fit in that size class. As a
+  // result, the maximum offset will be at most the maximum alignment for the
+  // last size class minus the header size, in multiples of MinAlignment.
+  UnpackedHeader Header = {};
+  const uptr MaxPrimaryAlignment =
+      1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment);
+  const uptr MaxOffset =
+      (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
+  Header.Offset = MaxOffset;
+  if (Header.Offset != MaxOffset)
+    dieWithMessage("maximum possible offset doesn't fit in header\n");
+  // Verify that we can fit the maximum size or amount of unused bytes in the
+  // header. Given that the Secondary fits the allocation to a page, the worst
+  // case scenario happens in the Primary. It will depend on the second to
+  // last and last class sizes, as well as the dynamic base for the Primary.
+  // The following is an over-approximation that works for our needs.
+  const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
+  Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
+  if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes)
+    dieWithMessage("maximum possible unused bytes doesn't fit in header\n");
+
+  const uptr LargestClassId = SizeClassMap::kLargestClassID;
+  Header.ClassId = LargestClassId;
+  if (Header.ClassId != LargestClassId)
+    dieWithMessage("largest class ID doesn't fit in header\n");
+}
 
-static ScudoBackendAllocator &getBackendAllocator() {
-  return Instance.BackendAllocator;
+// Opportunistic RSS limit check. This will update the RSS limit status, if
+// it can, every 100ms, otherwise it will just return the current one.
+NOINLINE bool Allocator::isRssLimitExceeded() {
+  u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
+  const u64 CurrentCheck = MonotonicNanoTime();
+  if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
+    return atomic_load_relaxed(&RssLimitExceeded);
+  if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
+                                    CurrentCheck, memory_order_relaxed))
+    return atomic_load_relaxed(&RssLimitExceeded);
+  // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the
+  //                RSS from /proc/self/statm by default. We might want to
+  //                call getrusage directly, even if it's less accurate.
+  const uptr CurrentRssMb = GetRSS() >> 20;
+  if (HardRssLimitMb && UNLIKELY(HardRssLimitMb < CurrentRssMb))
+    dieWithMessage("hard RSS limit exhausted (%zdMb vs %zdMb)\n",
+                   HardRssLimitMb, CurrentRssMb);
+  if (SoftRssLimitMb) {
+    if (atomic_load_relaxed(&RssLimitExceeded)) {
+      if (CurrentRssMb <= SoftRssLimitMb)
+        atomic_store_relaxed(&RssLimitExceeded, false);
+    } else {
+      if (CurrentRssMb > SoftRssLimitMb) {
+        atomic_store_relaxed(&RssLimitExceeded, true);
+        Printf("Scudo INFO: soft RSS limit exhausted (%zdMb vs %zdMb)\n",
+               SoftRssLimitMb, CurrentRssMb);
+      }
+    }
+  }
+  return atomic_load_relaxed(&RssLimitExceeded);
+}
+
+static Allocator Instance(LINKER_INITIALIZED);
+
+static BackendT &getBackend() {
+  return Instance.Backend;
 }
 
 void initScudo() {
   Instance.init();
 }
 
-void ScudoTSD::init(bool Shared) {
-  UnlockRequired = Shared;
-  getBackendAllocator().initCache(&Cache);
+void ScudoTSD::init() {
+  getBackend().initCache(&Cache);
   memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
 }
 
@@ -625,23 +641,25 @@ void ScudoTSD::commitBack() {
   Instance.commitBack(this);
 }
 
-void *scudoMalloc(uptr Size, AllocType Type) {
-  return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, Type));
-}
-
-void scudoFree(void *Ptr, AllocType Type) {
-  Instance.deallocate(Ptr, 0, Type);
+void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type) {
+  if (Alignment && UNLIKELY(!IsPowerOfTwo(Alignment))) {
+    errno = EINVAL;
+    if (Instance.canReturnNull())
+      return nullptr;
+    reportAllocationAlignmentNotPowerOfTwo(Alignment);
+  }
+  return SetErrnoOnNull(Instance.allocate(Size, Alignment, Type));
 }
 
-void scudoSizedFree(void *Ptr, uptr Size, AllocType Type) {
-  Instance.deallocate(Ptr, Size, Type);
+void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type) {
+  Instance.deallocate(Ptr, Size, Alignment, Type);
 }
 
 void *scudoRealloc(void *Ptr, uptr Size) {
   if (!Ptr)
     return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, FromMalloc));
   if (Size == 0) {
-    Instance.deallocate(Ptr, 0, FromMalloc);
+    Instance.deallocate(Ptr, 0, 0, FromMalloc);
     return nullptr;
   }
   return SetErrnoOnNull(Instance.reallocate(Ptr, Size));
@@ -660,24 +678,19 @@ void *scudoPvalloc(uptr Size) {
   uptr PageSize = GetPageSizeCached();
   if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) {
     errno = ENOMEM;
-    return Instance.handleBadRequest();
+    if (Instance.canReturnNull())
+      return nullptr;
+    reportPvallocOverflow(Size);
   }
   // pvalloc(0) should allocate one page.
   Size = Size ? RoundUpTo(Size, PageSize) : PageSize;
   return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign));
 }
 
-void *scudoMemalign(uptr Alignment, uptr Size) {
-  if (UNLIKELY(!IsPowerOfTwo(Alignment))) {
-    errno = EINVAL;
-    return Instance.handleBadRequest();
-  }
-  return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMemalign));
-}
-
 int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) {
   if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) {
-    Instance.handleBadRequest();
+    if (!Instance.canReturnNull())
+      reportInvalidPosixMemalignAlignment(Alignment);
     return EINVAL;
   }
   void *Ptr = Instance.allocate(Size, Alignment, FromMemalign);
@@ -690,7 +703,9 @@ int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) {
 void *scudoAlignedAlloc(uptr Alignment, uptr Size) {
   if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) {
     errno = EINVAL;
-    return Instance.handleBadRequest();
+    if (Instance.canReturnNull())
+      return nullptr;
+    reportInvalidAlignedAllocAlignment(Size, Alignment);
   }
   return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc));
 }
@@ -721,8 +736,8 @@ uptr __sanitizer_get_unmapped_bytes() {
   return 1;
 }
 
-uptr __sanitizer_get_estimated_allocated_size(uptr size) {
-  return size;
+uptr __sanitizer_get_estimated_allocated_size(uptr Size) {
+  return Size;
 }
 
 int __sanitizer_get_ownership(const void *Ptr) {
@@ -733,12 +748,26 @@ uptr __sanitizer_get_allocated_size(const void *Ptr) {
   return Instance.getUsableSize(Ptr);
 }
 
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook,
+                             void *Ptr, uptr Size) {
+  (void)Ptr;
+  (void)Size;
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *Ptr) {
+  (void)Ptr;
+}
+#endif
+
 // Interface functions
 
-extern "C" {
-void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit) {  // NOLINT
+void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit) {
   if (!SCUDO_CAN_USE_PUBLIC_INTERFACE)
     return;
   Instance.setRssLimit(LimitMb, !!HardLimit);
 }
-}  // extern "C"
+
+void __scudo_print_stats() {
+  Instance.printStats();
+}