diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h index 1ae463f845c81..4bdeb51873787 100644 --- a/compiler-rt/lib/hwasan/hwasan.h +++ b/compiler-rt/lib/hwasan/hwasan.h @@ -136,6 +136,21 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, StackTrace *stack); void hwasan_free(void *ptr, StackTrace *stack); +void *hwasan_new(uptr size, StackTrace *stack); +void *hwasan_new_aligned(uptr size, uptr alignment, StackTrace *stack); +void *hwasan_new_array(uptr size, StackTrace *stack); +void *hwasan_new_array_aligned(uptr size, uptr alignment, StackTrace *stack); +void hwasan_delete(void *ptr, StackTrace *stack); +void hwasan_delete_aligned(void *ptr, uptr alignment, StackTrace *stack); +void hwasan_delete_sized(void *ptr, uptr size, StackTrace *stack); +void hwasan_delete_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack); +void hwasan_delete_array(void *ptr, StackTrace *stack); +void hwasan_delete_array_aligned(void *ptr, uptr alignment, StackTrace *stack); +void hwasan_delete_array_sized(void *ptr, uptr size, StackTrace *stack); +void hwasan_delete_array_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack); + void InstallAtExitHandler(); #define GET_MALLOC_STACK_TRACE \ diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp index 75dbb336e3445..ec47be27f0e7e 100644 --- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp +++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp @@ -128,6 +128,12 @@ inline __lsan::ChunkTag Metadata::GetLsanTag() const { return static_cast<__lsan::ChunkTag>(lsan_tag); } +inline void Metadata::SetAllocType(AllocType type) { alloc_type = type; } + +inline AllocType Metadata::GetAllocType() const { + return static_cast(alloc_type); +} + uptr GetAliasRegionStart() { #if defined(HWASAN_ALIASING_MODE) constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); @@ -181,7 +187,7 @@ static uptr TaggedSize(uptr size) { } static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, - bool zeroise) { + AllocType alloc_type, bool zeroise) { // Keep this consistent with LSAN and ASAN behavior. if (UNLIKELY(orig_size == 0)) orig_size = 1; @@ -259,6 +265,7 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, meta->SetLsanTag(__lsan::DisabledInThisThread() ? __lsan::kIgnored : __lsan::kDirectlyLeaked); #endif + meta->SetAllocType(alloc_type); meta->SetAllocated(StackDepotPut(*stack), orig_size); RunMallocHooks(user_ptr, orig_size); return user_ptr; @@ -285,7 +292,7 @@ static bool CheckInvalidFree(StackTrace *stack, void *untagged_ptr, return false; } -static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { +static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr, AllocType) { CHECK(tagged_ptr); void *untagged_ptr = UntagPtr(tagged_ptr); @@ -379,15 +386,15 @@ static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old, void *untagged_ptr_old = UntagPtr(tagged_ptr_old); if (CheckInvalidFree(stack, untagged_ptr_old, tagged_ptr_old)) return nullptr; - void *tagged_ptr_new = - HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); + void *tagged_ptr_new = HwasanAllocate(stack, new_size, alignment, FROM_MALLOC, + false /*zeroise*/); if (tagged_ptr_old && tagged_ptr_new) { Metadata *meta = reinterpret_cast(allocator.GetMetaData(untagged_ptr_old)); void *untagged_ptr_new = UntagPtr(tagged_ptr_new); internal_memcpy(untagged_ptr_new, untagged_ptr_old, Min(new_size, static_cast(meta->GetRequestedSize()))); - HwasanDeallocate(stack, tagged_ptr_old); + HwasanDeallocate(stack, tagged_ptr_old, FROM_MALLOC); } return tagged_ptr_new; } @@ -398,7 +405,7 @@ static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) { return nullptr; ReportCallocOverflow(nmemb, size, stack); } - return HwasanAllocate(stack, nmemb * size, sizeof(u64), true); + return HwasanAllocate(stack, nmemb * size, sizeof(u64), FROM_MALLOC, true); } HwasanChunkView FindHeapChunkByAddress(uptr address) { @@ -449,7 +456,8 @@ static uptr AllocationSizeFast(const void *p) { } void *hwasan_malloc(uptr size, StackTrace *stack) { - return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, sizeof(u64), FROM_MALLOC, false)); } void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { @@ -458,9 +466,10 @@ void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { if (!ptr) - return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, sizeof(u64), FROM_MALLOC, false)); if (size == 0) { - HwasanDeallocate(stack, ptr); + HwasanDeallocate(stack, ptr, FROM_MALLOC); return nullptr; } return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); @@ -478,7 +487,7 @@ void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { void *hwasan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull( - HwasanAllocate(stack, size, GetPageSizeCached(), false)); + HwasanAllocate(stack, size, GetPageSizeCached(), FROM_MALLOC, false)); } void *hwasan_pvalloc(uptr size, StackTrace *stack) { @@ -491,7 +500,8 @@ void *hwasan_pvalloc(uptr size, StackTrace *stack) { } // pvalloc(0) should allocate one page. size = size ? RoundUpTo(size, PageSize) : PageSize; - return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, PageSize, FROM_MALLOC, false)); } void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) { @@ -501,7 +511,8 @@ void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) { return nullptr; ReportInvalidAlignedAllocAlignment(size, alignment, stack); } - return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, alignment, FROM_MALLOC, false)); } void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) { @@ -511,7 +522,8 @@ void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) { return nullptr; ReportInvalidAllocationAlignment(alignment, stack); } - return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, alignment, FROM_MALLOC, false)); } int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, @@ -521,7 +533,7 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, return errno_EINVAL; ReportInvalidPosixMemalignAlignment(alignment, stack); } - void *ptr = HwasanAllocate(stack, size, alignment, false); + void *ptr = HwasanAllocate(stack, size, alignment, FROM_MALLOC, false); if (UNLIKELY(!ptr)) // OOM error is already taken care of by HwasanAllocate. return errno_ENOMEM; @@ -531,7 +543,95 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, } void hwasan_free(void *ptr, StackTrace *stack) { - return HwasanDeallocate(stack, ptr); + HwasanDeallocate(stack, ptr, FROM_MALLOC); +} + +namespace { + +void *hwasan_new(uptr size, StackTrace *stack, bool array) { + return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), + array ? FROM_NEW_BR : FROM_NEW, true)); +} + +void *hwasan_new_aligned(uptr size, uptr alignment, StackTrace *stack, + bool array) { + if (UNLIKELY(alignment == 0 || !IsPowerOfTwo(alignment))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAllocationAlignment(alignment, stack); + } + return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, + array ? FROM_NEW_BR : FROM_NEW, false)); +} + +void hwasan_delete(void *ptr, StackTrace *stack, bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +void hwasan_delete_aligned(void *ptr, uptr, StackTrace *stack, bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +void hwasan_delete_sized(void *ptr, uptr, StackTrace *stack, bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +void hwasan_delete_sized_aligned(void *ptr, uptr, uptr, StackTrace *stack, + bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +} // namespace + +void *hwasan_new(uptr size, StackTrace *stack) { + return hwasan_new(size, stack, /*array=*/false); +} + +void *hwasan_new_aligned(uptr size, uptr alignment, StackTrace *stack) { + return hwasan_new_aligned(size, alignment, stack, /*array=*/false); +} + +void *hwasan_new_array(uptr size, StackTrace *stack) { + return hwasan_new(size, stack, /*array=*/true); +} + +void *hwasan_new_array_aligned(uptr size, uptr alignment, StackTrace *stack) { + return hwasan_new_aligned(size, alignment, stack, /*array=*/true); +} + +void hwasan_delete(void *ptr, StackTrace *stack) { + hwasan_delete(ptr, stack, /*array=*/false); +} + +void hwasan_delete_aligned(void *ptr, uptr alignment, StackTrace *stack) { + hwasan_delete_aligned(ptr, alignment, stack, /*array=*/false); +} + +void hwasan_delete_sized(void *ptr, uptr size, StackTrace *stack) { + hwasan_delete_sized(ptr, size, stack, /*array=*/false); +} + +void hwasan_delete_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack) { + hwasan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/false); +} + +void hwasan_delete_array(void *ptr, StackTrace *stack) { + hwasan_delete(ptr, stack, /*array=*/true); +} + +void hwasan_delete_array_aligned(void *ptr, uptr alignment, StackTrace *stack) { + hwasan_delete_aligned(ptr, alignment, stack, /*array=*/true); +} + +void hwasan_delete_array_sized(void *ptr, uptr size, StackTrace *stack) { + hwasan_delete_sized(ptr, size, stack, /*array=*/true); +} + +void hwasan_delete_array_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack) { + hwasan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/true); } } // namespace __hwasan diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h index 2ada2a0b1851a..6c28ffed591e6 100644 --- a/compiler-rt/lib/hwasan/hwasan_allocator.h +++ b/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -31,13 +31,20 @@ namespace __hwasan { +enum AllocType { + FROM_MALLOC = 1, // Memory block came from malloc, calloc, realloc, etc. + FROM_NEW = 2, // Memory block came from operator new. + FROM_NEW_BR = 3 // Memory block came from operator new [ ] +}; + struct Metadata { private: atomic_uint64_t alloc_context_id; u32 requested_size_low; u16 requested_size_high; atomic_uint8_t chunk_state; - u8 lsan_tag; + u8 lsan_tag : 2; + u8 alloc_type : 2; public: inline void SetAllocated(u32 stack, u64 size); @@ -49,6 +56,8 @@ struct Metadata { inline u32 GetAllocThreadId() const; inline void SetLsanTag(__lsan::ChunkTag tag); inline __lsan::ChunkTag GetLsanTag() const; + inline void SetAllocType(AllocType type); + inline AllocType GetAllocType() const; }; static_assert(sizeof(Metadata) == 16); diff --git a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp index f0fd3726ef1b1..3517364436bc2 100644 --- a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp +++ b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp @@ -22,23 +22,43 @@ #if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE // TODO(alekseys): throw std::bad_alloc instead of dying on OOM. -# define OPERATOR_NEW_BODY(nothrow) \ - GET_MALLOC_STACK_TRACE; \ - void *res = hwasan_malloc(size, &stack); \ - if (!nothrow && UNLIKELY(!res)) \ - ReportOutOfMemory(size, &stack); \ - return res -# define OPERATOR_NEW_ALIGN_BODY(nothrow) \ - GET_MALLOC_STACK_TRACE; \ - void *res = hwasan_memalign(static_cast(align), size, &stack); \ - if (!nothrow && UNLIKELY(!res)) \ - ReportOutOfMemory(size, &stack); \ - return res - -# define OPERATOR_DELETE_BODY \ - GET_MALLOC_STACK_TRACE; \ - if (ptr) \ - hwasan_free(ptr, &stack) +# define OPERATOR_NEW_BODY(array, nothrow) \ + GET_MALLOC_STACK_TRACE; \ + void *res = \ + array ? hwasan_new_array(size, &stack) : hwasan_new(size, &stack); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ + return res; +# define OPERATOR_NEW_BODY_ALIGN(array, nothrow) \ + GET_MALLOC_STACK_TRACE; \ + void *res = \ + array \ + ? hwasan_new_array_aligned(size, static_cast(align), &stack) \ + : hwasan_new_aligned(size, static_cast(align), &stack); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ + return res; + +# define OPERATOR_DELETE_BODY(array) \ + GET_MALLOC_STACK_TRACE; \ + array ? hwasan_delete_array(ptr, &stack) : hwasan_delete(ptr, &stack); + +# define OPERATOR_DELETE_BODY_SIZE(array) \ + GET_MALLOC_STACK_TRACE; \ + array ? hwasan_delete_array_sized(ptr, size, &stack) \ + : hwasan_delete_sized(ptr, size, &stack); + +# define OPERATOR_DELETE_BODY_ALIGN(array) \ + GET_MALLOC_STACK_TRACE; \ + array ? hwasan_delete_array_aligned(ptr, static_cast(align), &stack) \ + : hwasan_delete_aligned(ptr, static_cast(align), &stack); + +# define OPERATOR_DELETE_BODY_SIZE_ALIGN(array) \ + GET_MALLOC_STACK_TRACE; \ + array ? hwasan_delete_array_sized_aligned( \ + ptr, size, static_cast(align), &stack) \ + : hwasan_delete_sized_aligned(ptr, size, static_cast(align), \ + &stack); #elif defined(__ANDROID__) @@ -46,8 +66,9 @@ // since we previously released a runtime that intercepted these functions, // removing the interceptors would break ABI. Therefore we simply forward to // malloc and free. -# define OPERATOR_NEW_BODY(nothrow) return malloc(size) -# define OPERATOR_DELETE_BODY free(ptr) +# define OPERATOR_NEW_BODY(array, nothrow) return malloc(size) +# define OPERATOR_DELETE_BODY(array) free(ptr) +# define OPERATOR_DELETE_BODY_SIZE(array) free(ptr) #endif @@ -61,44 +82,44 @@ struct nothrow_t {}; } // namespace std INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(size_t size) { - OPERATOR_NEW_BODY(false /*nothrow*/); + OPERATOR_NEW_BODY(false /*array*/, false /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[]( size_t size) { - OPERATOR_NEW_BODY(false /*nothrow*/); + OPERATOR_NEW_BODY(true /*array*/, false /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new( size_t size, std::nothrow_t const &) { - OPERATOR_NEW_BODY(true /*nothrow*/); + OPERATOR_NEW_BODY(false /*array*/, true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[]( size_t size, std::nothrow_t const &) { - OPERATOR_NEW_BODY(true /*nothrow*/); + OPERATOR_NEW_BODY(true /*array*/, true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( void *ptr) NOEXCEPT { - OPERATOR_DELETE_BODY; + OPERATOR_DELETE_BODY(false /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( void *ptr) NOEXCEPT { - OPERATOR_DELETE_BODY; + OPERATOR_DELETE_BODY(true /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( void *ptr, std::nothrow_t const &) { - OPERATOR_DELETE_BODY; + OPERATOR_DELETE_BODY(false /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( void *ptr, std::nothrow_t const &) { - OPERATOR_DELETE_BODY; + OPERATOR_DELETE_BODY(true /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, size_t) NOEXCEPT { - OPERATOR_DELETE_BODY; + void *ptr, size_t size) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE(false /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, size_t) NOEXCEPT { - OPERATOR_DELETE_BODY; + void *ptr, size_t size) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE(true /*array*/); } #endif // OPERATOR_NEW_BODY @@ -111,52 +132,44 @@ enum class align_val_t : size_t {}; INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new( size_t size, std::align_val_t align) { - OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/); + OPERATOR_NEW_BODY_ALIGN(false /*array*/, false /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[]( size_t size, std::align_val_t align) { - OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/); + OPERATOR_NEW_BODY_ALIGN(true /*array*/, false /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new( size_t size, std::align_val_t align, std::nothrow_t const &) { - OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/); + OPERATOR_NEW_BODY_ALIGN(false /*array*/, true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[]( size_t size, std::align_val_t align, std::nothrow_t const &) { - OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/); + OPERATOR_NEW_BODY_ALIGN(true /*array*/, true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( void *ptr, std::align_val_t align) NOEXCEPT { - OPERATOR_DELETE_BODY; + OPERATOR_DELETE_BODY_ALIGN(false /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, std::align_val_t) NOEXCEPT { - OPERATOR_DELETE_BODY; -} -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT { - OPERATOR_DELETE_BODY; -} -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT { - OPERATOR_DELETE_BODY; + void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY_ALIGN(true /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, size_t, std::align_val_t) NOEXCEPT { - OPERATOR_DELETE_BODY; + void *ptr, std::align_val_t align, std::nothrow_t const &) { + OPERATOR_DELETE_BODY_ALIGN(false /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, size_t, std::align_val_t) NOEXCEPT { - OPERATOR_DELETE_BODY; + void *ptr, std::align_val_t align, std::nothrow_t const &) { + OPERATOR_DELETE_BODY_ALIGN(true /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, size_t, std::align_val_t, std::nothrow_t const &) NOEXCEPT { - OPERATOR_DELETE_BODY; + void *ptr, size_t size, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE_ALIGN(false /*array*/); } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, size_t, std::align_val_t, std::nothrow_t const &) NOEXCEPT { - OPERATOR_DELETE_BODY; + void *ptr, size_t size, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE_ALIGN(true /*array*/); } #endif // OPERATOR_NEW_ALIGN_BODY